2024-11-14 14:52:35,085 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-11-14 14:52:35,097 main DEBUG Took 0.010090 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 14:52:35,097 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 14:52:35,097 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 14:52:35,098 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 14:52:35,099 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:52:35,106 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 14:52:35,117 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:52:35,118 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:52:35,119 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:52:35,119 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:52:35,119 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:52:35,120 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:52:35,120 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:52:35,121 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:52:35,121 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:52:35,121 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:52:35,122 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:52:35,122 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:52:35,123 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:52:35,123 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:52:35,123 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:52:35,124 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:52:35,124 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:52:35,124 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:52:35,125 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:52:35,125 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:52:35,125 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:52:35,126 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:52:35,126 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:52:35,126 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 14:52:35,127 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:52:35,127 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 14:52:35,128 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 14:52:35,130 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 14:52:35,131 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 14:52:35,132 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 14:52:35,133 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 14:52:35,133 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 14:52:35,141 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 14:52:35,143 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 14:52:35,145 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 14:52:35,145 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 14:52:35,145 main DEBUG createAppenders(={Console}) 2024-11-14 14:52:35,146 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized 2024-11-14 14:52:35,146 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-11-14 14:52:35,147 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK. 2024-11-14 14:52:35,147 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 14:52:35,147 main DEBUG OutputStream closed 2024-11-14 14:52:35,148 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 14:52:35,148 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 14:52:35,148 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK 2024-11-14 14:52:35,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 14:52:35,214 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 14:52:35,215 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 14:52:35,215 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 14:52:35,216 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 14:52:35,216 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 14:52:35,216 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 14:52:35,217 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 14:52:35,217 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 14:52:35,217 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 14:52:35,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 14:52:35,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 14:52:35,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 14:52:35,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 14:52:35,219 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 14:52:35,219 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 14:52:35,219 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 14:52:35,220 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 14:52:35,222 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 14:52:35,222 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null 2024-11-14 14:52:35,223 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 14:52:35,223 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK. 2024-11-14T14:52:35,485 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d 2024-11-14 14:52:35,488 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 14:52:35,488 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T14:52:35,498 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins 2024-11-14T14:52:35,529 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=161, ProcessCount=11, AvailableMemoryMB=8591 2024-11-14T14:52:35,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T14:52:35,546 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/cluster_2aef49a5-0640-82c6-34f8-67754afcba77, deleteOnExit=true 2024-11-14T14:52:35,546 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T14:52:35,547 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/test.cache.data in system properties and HBase conf 2024-11-14T14:52:35,548 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T14:52:35,548 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/hadoop.log.dir in system properties and HBase conf 2024-11-14T14:52:35,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T14:52:35,549 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T14:52:35,550 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T14:52:35,638 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T14:52:35,725 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T14:52:35,729 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T14:52:35,729 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T14:52:35,730 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T14:52:35,730 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T14:52:35,730 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T14:52:35,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T14:52:35,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T14:52:35,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T14:52:35,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T14:52:35,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/nfs.dump.dir in system properties and HBase conf 2024-11-14T14:52:35,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/java.io.tmpdir in system properties and HBase conf 2024-11-14T14:52:35,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T14:52:35,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T14:52:35,734 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T14:52:36,237 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-14T14:52:36,569 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T14:52:36,652 INFO [Time-limited test {}] log.Log(170): Logging initialized @2272ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T14:52:36,730 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:52:36,795 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:52:36,814 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:52:36,815 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:52:36,816 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:52:36,828 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:52:36,830 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:52:36,831 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:52:37,024 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/java.io.tmpdir/jetty-localhost-35567-hadoop-hdfs-3_4_1-tests_jar-_-any-11237228005306852274/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T14:52:37,032 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:35567} 2024-11-14T14:52:37,032 INFO [Time-limited test {}] server.Server(415): Started @2654ms 2024-11-14T14:52:37,071 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-14T14:52:37,450 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:52:37,458 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:52:37,459 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:52:37,459 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:52:37,459 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:52:37,461 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:52:37,461 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:52:37,585 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/java.io.tmpdir/jetty-localhost-34461-hadoop-hdfs-3_4_1-tests_jar-_-any-9404139726985101996/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:52:37,586 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:34461} 2024-11-14T14:52:37,586 INFO [Time-limited test {}] server.Server(415): Started @3208ms 2024-11-14T14:52:37,646 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:52:37,773 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:52:37,780 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:52:37,784 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:52:37,784 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:52:37,785 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T14:52:37,786 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:52:37,787 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:52:37,926 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/java.io.tmpdir/jetty-localhost-45691-hadoop-hdfs-3_4_1-tests_jar-_-any-18427427206009681807/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:52:37,927 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:45691} 2024-11-14T14:52:37,927 INFO [Time-limited test {}] server.Server(415): Started @3549ms 2024-11-14T14:52:37,932 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:52:38,130 WARN [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/cluster_2aef49a5-0640-82c6-34f8-67754afcba77/data/data4/current/BP-494856361-172.17.0.2-1731595956337/current, will proceed with Du for space computation calculation, 2024-11-14T14:52:38,130 WARN [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/cluster_2aef49a5-0640-82c6-34f8-67754afcba77/data/data2/current/BP-494856361-172.17.0.2-1731595956337/current, will proceed with Du for space computation calculation, 2024-11-14T14:52:38,130 WARN [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/cluster_2aef49a5-0640-82c6-34f8-67754afcba77/data/data3/current/BP-494856361-172.17.0.2-1731595956337/current, will proceed with Du for space computation calculation, 2024-11-14T14:52:38,130 WARN [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/cluster_2aef49a5-0640-82c6-34f8-67754afcba77/data/data1/current/BP-494856361-172.17.0.2-1731595956337/current, will proceed with Du for space computation calculation, 2024-11-14T14:52:38,179 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:52:38,180 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:52:38,264 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x730a86ecd0e707a8 with lease ID 0x768a663fa8df6598: Processing first storage report for DS-45f05b47-c238-41c5-8883-f6522d93c5d1 from datanode DatanodeRegistration(127.0.0.1:46015, datanodeUuid=612b49a2-9a4b-47b8-a208-807e10ffe37f, infoPort=43375, infoSecurePort=0, ipcPort=41793, storageInfo=lv=-57;cid=testClusterID;nsid=2086092879;c=1731595956337) 2024-11-14T14:52:38,266 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x730a86ecd0e707a8 with lease ID 0x768a663fa8df6598: from storage DS-45f05b47-c238-41c5-8883-f6522d93c5d1 node DatanodeRegistration(127.0.0.1:46015, datanodeUuid=612b49a2-9a4b-47b8-a208-807e10ffe37f, infoPort=43375, infoSecurePort=0, ipcPort=41793, storageInfo=lv=-57;cid=testClusterID;nsid=2086092879;c=1731595956337), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-14T14:52:38,267 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb13e4a252c3e4f89 with lease ID 0x768a663fa8df6599: Processing first storage report for DS-91fa873e-0cc6-49ec-952a-034e227e82b8 from datanode DatanodeRegistration(127.0.0.1:43709, datanodeUuid=d6da9506-e7e9-4967-9b25-916232d8b6f9, infoPort=35365, infoSecurePort=0, ipcPort=37219, storageInfo=lv=-57;cid=testClusterID;nsid=2086092879;c=1731595956337) 2024-11-14T14:52:38,267 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb13e4a252c3e4f89 with lease ID 0x768a663fa8df6599: from storage DS-91fa873e-0cc6-49ec-952a-034e227e82b8 node DatanodeRegistration(127.0.0.1:43709, datanodeUuid=d6da9506-e7e9-4967-9b25-916232d8b6f9, infoPort=35365, infoSecurePort=0, ipcPort=37219, storageInfo=lv=-57;cid=testClusterID;nsid=2086092879;c=1731595956337), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:52:38,267 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x730a86ecd0e707a8 with lease ID 0x768a663fa8df6598: Processing first storage report for DS-32191710-7e57-4daa-be07-85b3cc85b77a from datanode DatanodeRegistration(127.0.0.1:46015, datanodeUuid=612b49a2-9a4b-47b8-a208-807e10ffe37f, infoPort=43375, infoSecurePort=0, ipcPort=41793, storageInfo=lv=-57;cid=testClusterID;nsid=2086092879;c=1731595956337) 2024-11-14T14:52:38,268 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x730a86ecd0e707a8 with lease ID 0x768a663fa8df6598: from storage DS-32191710-7e57-4daa-be07-85b3cc85b77a node DatanodeRegistration(127.0.0.1:46015, datanodeUuid=612b49a2-9a4b-47b8-a208-807e10ffe37f, infoPort=43375, infoSecurePort=0, ipcPort=41793, storageInfo=lv=-57;cid=testClusterID;nsid=2086092879;c=1731595956337), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:52:38,268 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb13e4a252c3e4f89 with lease ID 0x768a663fa8df6599: Processing first storage report for DS-5f9c46d7-79a2-40c1-af36-689d0b5f2e4e from datanode DatanodeRegistration(127.0.0.1:43709, datanodeUuid=d6da9506-e7e9-4967-9b25-916232d8b6f9, infoPort=35365, infoSecurePort=0, ipcPort=37219, storageInfo=lv=-57;cid=testClusterID;nsid=2086092879;c=1731595956337) 2024-11-14T14:52:38,268 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb13e4a252c3e4f89 with lease ID 0x768a663fa8df6599: from storage DS-5f9c46d7-79a2-40c1-af36-689d0b5f2e4e node DatanodeRegistration(127.0.0.1:43709, datanodeUuid=d6da9506-e7e9-4967-9b25-916232d8b6f9, infoPort=35365, infoSecurePort=0, ipcPort=37219, storageInfo=lv=-57;cid=testClusterID;nsid=2086092879;c=1731595956337), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:52:38,332 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d 2024-11-14T14:52:38,408 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/cluster_2aef49a5-0640-82c6-34f8-67754afcba77/zookeeper_0, clientPort=64763, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/cluster_2aef49a5-0640-82c6-34f8-67754afcba77/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/cluster_2aef49a5-0640-82c6-34f8-67754afcba77/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T14:52:38,419 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64763 2024-11-14T14:52:38,429 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:52:38,432 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:52:38,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:52:38,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:52:39,073 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a with version=8 2024-11-14T14:52:39,073 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/hbase-staging 2024-11-14T14:52:39,163 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T14:52:39,412 INFO [Time-limited test {}] client.ConnectionUtils(128): master/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:52:39,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:52:39,424 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:52:39,428 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:52:39,428 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:52:39,428 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:52:39,561 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T14:52:39,620 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T14:52:39,628 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T14:52:39,632 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:52:39,660 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 6054 (auto-detected) 2024-11-14T14:52:39,661 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-14T14:52:39,681 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40347 2024-11-14T14:52:39,702 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40347 connecting to ZooKeeper ensemble=127.0.0.1:64763 2024-11-14T14:52:39,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:403470x0, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:52:39,735 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40347-0x101a744eddd0000 connected 2024-11-14T14:52:39,759 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:52:39,762 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:52:39,772 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:52:39,776 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a, hbase.cluster.distributed=false 2024-11-14T14:52:39,798 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:52:39,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40347 2024-11-14T14:52:39,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40347 2024-11-14T14:52:39,803 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40347 2024-11-14T14:52:39,803 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40347 2024-11-14T14:52:39,804 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40347 2024-11-14T14:52:39,912 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:52:39,914 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:52:39,915 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:52:39,915 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:52:39,915 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:52:39,916 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:52:39,920 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T14:52:39,923 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:52:39,924 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33427 2024-11-14T14:52:39,927 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33427 connecting to ZooKeeper ensemble=127.0.0.1:64763 2024-11-14T14:52:39,928 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:52:39,934 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:52:39,941 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:334270x0, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:52:39,944 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33427-0x101a744eddd0001 connected 2024-11-14T14:52:39,944 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:52:39,949 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T14:52:39,959 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T14:52:39,961 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T14:52:39,967 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:52:39,971 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33427 2024-11-14T14:52:39,972 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33427 2024-11-14T14:52:39,973 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33427 2024-11-14T14:52:39,973 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33427 2024-11-14T14:52:39,974 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33427 2024-11-14T14:52:39,990 DEBUG [M:0;da1c6afcd1f9:40347 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;da1c6afcd1f9:40347 2024-11-14T14:52:39,991 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/da1c6afcd1f9,40347,1731595959216 2024-11-14T14:52:40,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:52:40,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:52:40,001 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/da1c6afcd1f9,40347,1731595959216 2024-11-14T14:52:40,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:52:40,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T14:52:40,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:52:40,026 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T14:52:40,027 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/da1c6afcd1f9,40347,1731595959216 from backup master directory 2024-11-14T14:52:40,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/da1c6afcd1f9,40347,1731595959216 2024-11-14T14:52:40,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:52:40,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:52:40,031 WARN [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:52:40,031 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=da1c6afcd1f9,40347,1731595959216 2024-11-14T14:52:40,033 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T14:52:40,035 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T14:52:40,095 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/hbase.id] with ID: 4b0401d9-b760-4bf9-8e6c-07c70687997b 2024-11-14T14:52:40,095 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/.tmp/hbase.id 2024-11-14T14:52:40,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:52:40,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:52:40,122 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/.tmp/hbase.id]:[hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/hbase.id] 2024-11-14T14:52:40,172 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:52:40,179 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T14:52:40,198 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-14T14:52:40,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:52:40,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:52:40,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:52:40,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:52:40,235 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T14:52:40,237 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T14:52:40,243 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:52:40,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:52:40,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:52:40,293 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store 2024-11-14T14:52:40,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:52:40,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:52:40,322 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T14:52:40,325 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:40,326 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:52:40,326 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:52:40,326 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:52:40,328 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:52:40,328 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:52:40,328 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:52:40,329 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731595960326Disabling compacts and flushes for region at 1731595960326Disabling writes for close at 1731595960328 (+2 ms)Writing region close event to WAL at 1731595960328Closed at 1731595960328 2024-11-14T14:52:40,331 WARN [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/.initializing 2024-11-14T14:52:40,331 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/WALs/da1c6afcd1f9,40347,1731595959216 2024-11-14T14:52:40,354 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C40347%2C1731595959216, suffix=, logDir=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/WALs/da1c6afcd1f9,40347,1731595959216, archiveDir=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/oldWALs, maxLogs=10 2024-11-14T14:52:40,363 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C40347%2C1731595959216.1731595960359 2024-11-14T14:52:40,381 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/WALs/da1c6afcd1f9,40347,1731595959216/da1c6afcd1f9%2C40347%2C1731595959216.1731595960359 2024-11-14T14:52:40,390 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43375:43375),(127.0.0.1/127.0.0.1:35365:35365)] 2024-11-14T14:52:40,392 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:40,392 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:40,395 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:52:40,396 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:52:40,444 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:52:40,477 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T14:52:40,481 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:40,485 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:52:40,485 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:52:40,489 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T14:52:40,489 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:40,490 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:40,491 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:52:40,493 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T14:52:40,494 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:40,495 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:40,495 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:52:40,497 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T14:52:40,497 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:40,498 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:40,499 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:52:40,502 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:52:40,503 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:52:40,509 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:52:40,510 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:52:40,515 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T14:52:40,520 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:52:40,526 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:52:40,528 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=773341, jitterRate=-0.016646191477775574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T14:52:40,538 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731595960408Initializing all the Stores at 1731595960411 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595960411Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595960412 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595960413 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595960413Cleaning up temporary data from old regions at 1731595960510 (+97 ms)Region opened successfully at 1731595960538 (+28 ms) 2024-11-14T14:52:40,540 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T14:52:40,589 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0a740a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:52:40,630 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T14:52:40,641 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T14:52:40,641 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T14:52:40,644 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:52:40,646 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T14:52:40,650 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-14T14:52:40,651 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T14:52:40,676 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T14:52:40,688 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T14:52:40,691 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T14:52:40,694 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T14:52:40,696 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T14:52:40,698 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T14:52:40,700 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T14:52:40,704 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T14:52:40,706 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T14:52:40,708 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T14:52:40,709 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T14:52:40,727 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T14:52:40,729 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T14:52:40,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:52:40,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:52:40,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:52:40,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:52:40,735 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=da1c6afcd1f9,40347,1731595959216, sessionid=0x101a744eddd0000, setting cluster-up flag (Was=false) 2024-11-14T14:52:40,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:52:40,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:52:40,755 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T14:52:40,757 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da1c6afcd1f9,40347,1731595959216 2024-11-14T14:52:40,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:52:40,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:52:40,768 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T14:52:40,770 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da1c6afcd1f9,40347,1731595959216 2024-11-14T14:52:40,778 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T14:52:40,857 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:52:40,870 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T14:52:40,876 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T14:52:40,878 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.HRegionServer(746): ClusterId : 4b0401d9-b760-4bf9-8e6c-07c70687997b 2024-11-14T14:52:40,881 DEBUG [RS:0;da1c6afcd1f9:33427 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T14:52:40,882 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: da1c6afcd1f9,40347,1731595959216 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T14:52:40,887 DEBUG [RS:0;da1c6afcd1f9:33427 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T14:52:40,887 DEBUG [RS:0;da1c6afcd1f9:33427 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T14:52:40,889 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:52:40,889 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:52:40,889 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:52:40,890 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:52:40,890 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/da1c6afcd1f9:0, corePoolSize=10, maxPoolSize=10 2024-11-14T14:52:40,890 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:52:40,890 DEBUG [RS:0;da1c6afcd1f9:33427 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T14:52:40,890 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:52:40,890 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:52:40,891 DEBUG [RS:0;da1c6afcd1f9:33427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a4b803d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:52:40,898 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731595990898 2024-11-14T14:52:40,900 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T14:52:40,900 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:52:40,901 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T14:52:40,901 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T14:52:40,906 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T14:52:40,906 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T14:52:40,906 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T14:52:40,906 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T14:52:40,908 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:40,908 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T14:52:40,908 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:40,911 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T14:52:40,912 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T14:52:40,913 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T14:52:40,913 DEBUG [RS:0;da1c6afcd1f9:33427 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;da1c6afcd1f9:33427 2024-11-14T14:52:40,917 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T14:52:40,917 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T14:52:40,917 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T14:52:40,917 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T14:52:40,917 DEBUG [RS:0;da1c6afcd1f9:33427 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T14:52:40,920 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731595960919,5,FailOnTimeoutGroup] 2024-11-14T14:52:40,920 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.HRegionServer(2659): reportForDuty to master=da1c6afcd1f9,40347,1731595959216 with port=33427, startcode=1731595959873 2024-11-14T14:52:40,921 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731595960920,5,FailOnTimeoutGroup] 2024-11-14T14:52:40,921 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:40,921 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T14:52:40,923 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:40,923 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:40,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:52:40,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:52:40,929 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T14:52:40,930 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a 2024-11-14T14:52:40,934 DEBUG [RS:0;da1c6afcd1f9:33427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T14:52:40,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:52:40,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:52:40,950 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:40,956 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:52:40,960 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:52:40,960 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:40,961 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:52:40,961 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:52:40,964 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:52:40,964 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:40,965 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:52:40,965 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:52:40,968 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:52:40,968 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:40,969 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:52:40,969 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:52:40,972 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:52:40,972 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:40,973 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:52:40,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:52:40,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740 2024-11-14T14:52:40,976 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740 2024-11-14T14:52:40,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:52:40,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:52:40,982 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T14:52:40,986 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:52:40,993 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:52:40,994 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=807012, jitterRate=0.026169508695602417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T14:52:40,997 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731595960951Initializing all the Stores at 1731595960952 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595960952Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595960956 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595960956Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595960956Cleaning up temporary data from old regions at 1731595960980 (+24 ms)Region opened successfully at 1731595960997 (+17 ms) 2024-11-14T14:52:40,997 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:52:40,998 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:52:40,998 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:52:40,998 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:52:40,998 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:52:41,000 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:52:41,001 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731595960997Disabling compacts and flushes for region at 1731595960997Disabling writes for close at 1731595960998 (+1 ms)Writing region close event to WAL at 1731595961000 (+2 ms)Closed at 1731595961000 2024-11-14T14:52:41,004 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:52:41,004 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T14:52:41,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T14:52:41,018 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52127, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T14:52:41,025 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:52:41,026 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40347 {}] master.ServerManager(363): Checking decommissioned status of RegionServer da1c6afcd1f9,33427,1731595959873 2024-11-14T14:52:41,029 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T14:52:41,030 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40347 {}] master.ServerManager(517): Registering regionserver=da1c6afcd1f9,33427,1731595959873 2024-11-14T14:52:41,045 DEBUG [RS:0;da1c6afcd1f9:33427 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a 2024-11-14T14:52:41,045 DEBUG [RS:0;da1c6afcd1f9:33427 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37399 2024-11-14T14:52:41,045 DEBUG [RS:0;da1c6afcd1f9:33427 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T14:52:41,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:52:41,050 DEBUG [RS:0;da1c6afcd1f9:33427 {}] zookeeper.ZKUtil(111): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/da1c6afcd1f9,33427,1731595959873 2024-11-14T14:52:41,051 WARN [RS:0;da1c6afcd1f9:33427 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:52:41,051 INFO [RS:0;da1c6afcd1f9:33427 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:52:41,051 DEBUG [RS:0;da1c6afcd1f9:33427 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873 2024-11-14T14:52:41,053 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [da1c6afcd1f9,33427,1731595959873] 2024-11-14T14:52:41,082 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T14:52:41,097 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T14:52:41,104 INFO [RS:0;da1c6afcd1f9:33427 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T14:52:41,104 INFO [RS:0;da1c6afcd1f9:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:41,105 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T14:52:41,112 INFO [RS:0;da1c6afcd1f9:33427 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T14:52:41,113 INFO [RS:0;da1c6afcd1f9:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:41,114 DEBUG [RS:0;da1c6afcd1f9:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:52:41,114 DEBUG [RS:0;da1c6afcd1f9:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:52:41,114 DEBUG [RS:0;da1c6afcd1f9:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:52:41,114 DEBUG [RS:0;da1c6afcd1f9:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:52:41,114 DEBUG [RS:0;da1c6afcd1f9:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:52:41,114 DEBUG [RS:0;da1c6afcd1f9:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:52:41,115 DEBUG [RS:0;da1c6afcd1f9:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:52:41,115 DEBUG [RS:0;da1c6afcd1f9:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:52:41,115 DEBUG [RS:0;da1c6afcd1f9:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:52:41,115 DEBUG [RS:0;da1c6afcd1f9:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:52:41,115 DEBUG [RS:0;da1c6afcd1f9:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:52:41,115 DEBUG [RS:0;da1c6afcd1f9:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:52:41,115 DEBUG [RS:0;da1c6afcd1f9:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:52:41,115 DEBUG [RS:0;da1c6afcd1f9:33427 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:52:41,118 INFO [RS:0;da1c6afcd1f9:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:41,118 INFO [RS:0;da1c6afcd1f9:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:41,118 INFO [RS:0;da1c6afcd1f9:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:41,118 INFO [RS:0;da1c6afcd1f9:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:41,118 INFO [RS:0;da1c6afcd1f9:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:41,119 INFO [RS:0;da1c6afcd1f9:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,33427,1731595959873-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:52:41,140 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T14:52:41,142 INFO [RS:0;da1c6afcd1f9:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,33427,1731595959873-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:41,142 INFO [RS:0;da1c6afcd1f9:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:41,143 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.Replication(171): da1c6afcd1f9,33427,1731595959873 started 2024-11-14T14:52:41,163 INFO [RS:0;da1c6afcd1f9:33427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:41,163 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.HRegionServer(1482): Serving as da1c6afcd1f9,33427,1731595959873, RpcServer on da1c6afcd1f9/172.17.0.2:33427, sessionid=0x101a744eddd0001 2024-11-14T14:52:41,164 DEBUG [RS:0;da1c6afcd1f9:33427 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T14:52:41,164 DEBUG [RS:0;da1c6afcd1f9:33427 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager da1c6afcd1f9,33427,1731595959873 2024-11-14T14:52:41,164 DEBUG [RS:0;da1c6afcd1f9:33427 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,33427,1731595959873' 2024-11-14T14:52:41,164 DEBUG [RS:0;da1c6afcd1f9:33427 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T14:52:41,166 DEBUG [RS:0;da1c6afcd1f9:33427 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T14:52:41,166 DEBUG [RS:0;da1c6afcd1f9:33427 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T14:52:41,167 DEBUG [RS:0;da1c6afcd1f9:33427 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T14:52:41,167 DEBUG [RS:0;da1c6afcd1f9:33427 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager da1c6afcd1f9,33427,1731595959873 2024-11-14T14:52:41,167 DEBUG [RS:0;da1c6afcd1f9:33427 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,33427,1731595959873' 2024-11-14T14:52:41,167 DEBUG [RS:0;da1c6afcd1f9:33427 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T14:52:41,168 DEBUG [RS:0;da1c6afcd1f9:33427 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T14:52:41,169 DEBUG [RS:0;da1c6afcd1f9:33427 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T14:52:41,169 INFO [RS:0;da1c6afcd1f9:33427 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T14:52:41,169 INFO [RS:0;da1c6afcd1f9:33427 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T14:52:41,180 WARN [da1c6afcd1f9:40347 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-14T14:52:41,278 INFO [RS:0;da1c6afcd1f9:33427 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C33427%2C1731595959873, suffix=, logDir=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873, archiveDir=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/oldWALs, maxLogs=32 2024-11-14T14:52:41,281 INFO [RS:0;da1c6afcd1f9:33427 {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C33427%2C1731595959873.1731595961280 2024-11-14T14:52:41,290 INFO [RS:0;da1c6afcd1f9:33427 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731595961280 2024-11-14T14:52:41,296 DEBUG [RS:0;da1c6afcd1f9:33427 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35365:35365),(127.0.0.1/127.0.0.1:43375:43375)] 2024-11-14T14:52:41,432 DEBUG [da1c6afcd1f9:40347 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T14:52:41,445 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=da1c6afcd1f9,33427,1731595959873 2024-11-14T14:52:41,452 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,33427,1731595959873, state=OPENING 2024-11-14T14:52:41,458 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T14:52:41,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:52:41,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:52:41,461 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:52:41,462 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:52:41,463 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:52:41,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,33427,1731595959873}] 2024-11-14T14:52:41,646 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T14:52:41,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36105, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T14:52:41,667 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T14:52:41,668 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:52:41,672 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C33427%2C1731595959873.meta, suffix=.meta, logDir=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873, archiveDir=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/oldWALs, maxLogs=32 2024-11-14T14:52:41,675 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C33427%2C1731595959873.meta.1731595961675.meta 2024-11-14T14:52:41,690 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.meta.1731595961675.meta 2024-11-14T14:52:41,692 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35365:35365),(127.0.0.1/127.0.0.1:43375:43375)] 2024-11-14T14:52:41,696 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:41,698 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T14:52:41,701 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T14:52:41,706 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T14:52:41,711 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T14:52:41,712 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:41,712 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T14:52:41,712 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T14:52:41,716 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:52:41,718 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:52:41,719 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:41,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:52:41,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:52:41,722 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:52:41,722 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:41,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:52:41,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:52:41,725 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:52:41,725 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:41,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:52:41,727 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:52:41,728 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:52:41,728 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:41,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:52:41,730 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:52:41,731 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740 2024-11-14T14:52:41,734 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740 2024-11-14T14:52:41,737 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:52:41,737 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:52:41,739 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T14:52:41,742 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:52:41,744 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=692349, jitterRate=-0.11963295936584473}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T14:52:41,745 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T14:52:41,747 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731595961713Writing region info on filesystem at 1731595961713Initializing all the Stores at 1731595961715 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595961715Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595961716 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595961716Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731595961716Cleaning up temporary data from old regions at 1731595961737 (+21 ms)Running coprocessor post-open hooks at 1731595961745 (+8 ms)Region opened successfully at 1731595961747 (+2 ms) 2024-11-14T14:52:41,755 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731595961635 2024-11-14T14:52:41,770 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T14:52:41,770 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T14:52:41,772 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,33427,1731595959873 2024-11-14T14:52:41,774 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,33427,1731595959873, state=OPEN 2024-11-14T14:52:41,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:52:41,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:52:41,781 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:52:41,781 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:52:41,781 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,33427,1731595959873 2024-11-14T14:52:41,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T14:52:41,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,33427,1731595959873 in 316 msec 2024-11-14T14:52:41,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T14:52:41,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 778 msec 2024-11-14T14:52:41,807 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:52:41,807 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T14:52:41,837 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:52:41,839 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,33427,1731595959873, seqNum=-1] 2024-11-14T14:52:41,867 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:52:41,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55945, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:52:41,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0800 sec 2024-11-14T14:52:41,891 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731595961891, completionTime=-1 2024-11-14T14:52:41,894 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T14:52:41,894 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:52:41,923 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-14T14:52:41,923 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731596021923 2024-11-14T14:52:41,923 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731596081923 2024-11-14T14:52:41,923 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-11-14T14:52:41,926 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40347,1731595959216-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:41,927 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40347,1731595959216-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:41,927 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40347,1731595959216-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:41,928 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-da1c6afcd1f9:40347, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:41,929 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:41,929 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:41,935 DEBUG [master/da1c6afcd1f9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T14:52:41,963 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.931sec 2024-11-14T14:52:41,965 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T14:52:41,966 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T14:52:41,967 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T14:52:41,967 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T14:52:41,968 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T14:52:41,969 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40347,1731595959216-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:52:41,969 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40347,1731595959216-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T14:52:41,978 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T14:52:41,979 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T14:52:41,979 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40347,1731595959216-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:52:41,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b598c24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:52:41,997 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T14:52:41,997 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T14:52:42,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request da1c6afcd1f9,40347,-1 for getting cluster id 2024-11-14T14:52:42,007 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T14:52:42,017 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b0401d9-b760-4bf9-8e6c-07c70687997b' 2024-11-14T14:52:42,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T14:52:42,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b0401d9-b760-4bf9-8e6c-07c70687997b" 2024-11-14T14:52:42,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38e17980, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:52:42,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [da1c6afcd1f9,40347,-1] 2024-11-14T14:52:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T14:52:42,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:52:42,028 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T14:52:42,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e15a4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:52:42,033 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:52:42,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,33427,1731595959873, seqNum=-1] 2024-11-14T14:52:42,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:52:42,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:52:42,070 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=da1c6afcd1f9,40347,1731595959216 2024-11-14T14:52:42,071 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:52:42,079 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-14T14:52:42,084 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T14:52:42,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is da1c6afcd1f9,40347,1731595959216 2024-11-14T14:52:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1f4e8374 2024-11-14T14:52:42,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T14:52:42,099 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T14:52:42,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40347 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-14T14:52:42,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40347 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-14T14:52:42,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40347 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T14:52:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40347 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling 2024-11-14T14:52:42,122 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T14:52:42,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40347 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testSlowSyncLogRolling" procId is: 4 2024-11-14T14:52:42,125 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:42,128 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T14:52:42,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:52:42,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741835_1011 (size=389) 2024-11-14T14:52:42,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741835_1011 (size=389) 2024-11-14T14:52:42,178 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 92807f1e92a275f371708d055eca2bfe, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a 2024-11-14T14:52:42,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741836_1012 (size=72) 2024-11-14T14:52:42,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741836_1012 (size=72) 2024-11-14T14:52:42,191 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:42,191 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 92807f1e92a275f371708d055eca2bfe, disabling compactions & flushes 2024-11-14T14:52:42,191 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe. 2024-11-14T14:52:42,191 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe. 2024-11-14T14:52:42,191 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe. after waiting 0 ms 2024-11-14T14:52:42,191 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe. 2024-11-14T14:52:42,191 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe. 2024-11-14T14:52:42,191 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 92807f1e92a275f371708d055eca2bfe: Waiting for close lock at 1731595962191Disabling compacts and flushes for region at 1731595962191Disabling writes for close at 1731595962191Writing region close event to WAL at 1731595962191Closed at 1731595962191 2024-11-14T14:52:42,194 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T14:52:42,199 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731595962194"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731595962194"}]},"ts":"1731595962194"} 2024-11-14T14:52:42,206 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T14:52:42,208 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T14:52:42,212 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731595962209"}]},"ts":"1731595962209"} 2024-11-14T14:52:42,217 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta 2024-11-14T14:52:42,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=92807f1e92a275f371708d055eca2bfe, ASSIGN}] 2024-11-14T14:52:42,223 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=92807f1e92a275f371708d055eca2bfe, ASSIGN 2024-11-14T14:52:42,226 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=92807f1e92a275f371708d055eca2bfe, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,33427,1731595959873; forceNewPlan=false, retain=false 2024-11-14T14:52:42,377 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=92807f1e92a275f371708d055eca2bfe, regionState=OPENING, regionLocation=da1c6afcd1f9,33427,1731595959873 2024-11-14T14:52:42,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=92807f1e92a275f371708d055eca2bfe, ASSIGN because future has completed 2024-11-14T14:52:42,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92807f1e92a275f371708d055eca2bfe, server=da1c6afcd1f9,33427,1731595959873}] 2024-11-14T14:52:42,546 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe. 2024-11-14T14:52:42,546 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 92807f1e92a275f371708d055eca2bfe, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:52:42,547 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 92807f1e92a275f371708d055eca2bfe 2024-11-14T14:52:42,547 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:52:42,547 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 92807f1e92a275f371708d055eca2bfe 2024-11-14T14:52:42,547 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 92807f1e92a275f371708d055eca2bfe 2024-11-14T14:52:42,550 INFO [StoreOpener-92807f1e92a275f371708d055eca2bfe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 92807f1e92a275f371708d055eca2bfe 2024-11-14T14:52:42,552 INFO [StoreOpener-92807f1e92a275f371708d055eca2bfe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92807f1e92a275f371708d055eca2bfe columnFamilyName info 2024-11-14T14:52:42,552 DEBUG [StoreOpener-92807f1e92a275f371708d055eca2bfe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:52:42,553 INFO [StoreOpener-92807f1e92a275f371708d055eca2bfe-1 {}] regionserver.HStore(327): Store=92807f1e92a275f371708d055eca2bfe/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:52:42,553 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 92807f1e92a275f371708d055eca2bfe 2024-11-14T14:52:42,555 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe 2024-11-14T14:52:42,556 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe 2024-11-14T14:52:42,557 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 92807f1e92a275f371708d055eca2bfe 2024-11-14T14:52:42,557 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 92807f1e92a275f371708d055eca2bfe 2024-11-14T14:52:42,559 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 92807f1e92a275f371708d055eca2bfe 2024-11-14T14:52:42,563 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:52:42,564 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 92807f1e92a275f371708d055eca2bfe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=691987, jitterRate=-0.12009373307228088}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:52:42,564 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 92807f1e92a275f371708d055eca2bfe 2024-11-14T14:52:42,565 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 92807f1e92a275f371708d055eca2bfe: Running coprocessor pre-open hook at 1731595962547Writing region info on filesystem at 1731595962547Initializing all the Stores at 1731595962549 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731595962549Cleaning up temporary data from old regions at 1731595962557 (+8 ms)Running coprocessor post-open hooks at 1731595962564 (+7 ms)Region opened successfully at 1731595962565 (+1 ms) 2024-11-14T14:52:42,568 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe., pid=6, masterSystemTime=1731595962539 2024-11-14T14:52:42,573 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe. 2024-11-14T14:52:42,573 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe. 2024-11-14T14:52:42,574 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=92807f1e92a275f371708d055eca2bfe, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,33427,1731595959873 2024-11-14T14:52:42,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92807f1e92a275f371708d055eca2bfe, server=da1c6afcd1f9,33427,1731595959873 because future has completed 2024-11-14T14:52:42,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T14:52:42,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 92807f1e92a275f371708d055eca2bfe, server=da1c6afcd1f9,33427,1731595959873 in 196 msec 2024-11-14T14:52:42,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T14:52:42,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=92807f1e92a275f371708d055eca2bfe, ASSIGN in 366 msec 2024-11-14T14:52:42,591 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T14:52:42,591 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731595962591"}]},"ts":"1731595962591"} 2024-11-14T14:52:42,595 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta 2024-11-14T14:52:42,597 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T14:52:42,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 485 msec 2024-11-14T14:52:47,216 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T14:52:47,283 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T14:52:47,285 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling' 2024-11-14T14:52:49,617 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T14:52:49,618 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T14:52:49,620 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-11-14T14:52:49,620 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer 2024-11-14T14:52:49,622 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:52:49,622 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T14:52:49,623 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T14:52:49,623 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T14:52:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40347 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:52:52,228 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed 2024-11-14T14:52:52,232 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100 2024-11-14T14:52:52,239 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling 2024-11-14T14:52:52,239 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe. 2024-11-14T14:52:52,241 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C33427%2C1731595959873.1731595972240 2024-11-14T14:52:52,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:52:52,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:52:52,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:52:52,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:52:52,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:52:52,260 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731595961280 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731595972240 2024-11-14T14:52:52,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741833_1009 (size=451) 2024-11-14T14:52:52,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741833_1009 (size=451) 2024-11-14T14:52:52,269 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731595961280 to hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/oldWALs/da1c6afcd1f9%2C33427%2C1731595959873.1731595961280 2024-11-14T14:52:52,272 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43375:43375),(127.0.0.1/127.0.0.1:35365:35365)] 2024-11-14T14:52:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe., hostname=da1c6afcd1f9,33427,1731595959873, seqNum=2] 2024-11-14T14:53:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33427 {}] regionserver.HRegion(8855): Flush requested on 92807f1e92a275f371708d055eca2bfe 2024-11-14T14:53:04,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92807f1e92a275f371708d055eca2bfe 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-14T14:53:04,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/.tmp/info/e8a0dfe627684057a735bfb2dd37efa2 is 1080, key is row0001/info:/1731595972288/Put/seqid=0 2024-11-14T14:53:04,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741838_1014 (size=12509) 2024-11-14T14:53:04,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741838_1014 (size=12509) 2024-11-14T14:53:04,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/.tmp/info/e8a0dfe627684057a735bfb2dd37efa2 2024-11-14T14:53:04,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/.tmp/info/e8a0dfe627684057a735bfb2dd37efa2 as hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/e8a0dfe627684057a735bfb2dd37efa2 2024-11-14T14:53:04,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/e8a0dfe627684057a735bfb2dd37efa2, entries=7, sequenceid=11, filesize=12.2 K 2024-11-14T14:53:04,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 92807f1e92a275f371708d055eca2bfe in 171ms, sequenceid=11, compaction requested=false 2024-11-14T14:53:04,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92807f1e92a275f371708d055eca2bfe: 2024-11-14T14:53:08,329 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T14:53:12,335 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C33427%2C1731595959873.1731595992334 2024-11-14T14:53:12,543 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:12,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:12,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:12,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:12,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:12,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:12,545 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731595972240 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731595992334 2024-11-14T14:53:12,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741837_1013 (size=12399) 2024-11-14T14:53:12,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741837_1013 (size=12399) 2024-11-14T14:53:12,549 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43375:43375),(127.0.0.1/127.0.0.1:35365:35365)] 2024-11-14T14:53:12,549 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731595972240 is not closed yet, will try archiving it next time 2024-11-14T14:53:12,752 INFO [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:14,956 INFO [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:17,160 INFO [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:19,364 INFO [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33427 {}] regionserver.HRegion(8855): Flush requested on 92807f1e92a275f371708d055eca2bfe 2024-11-14T14:53:19,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92807f1e92a275f371708d055eca2bfe 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-14T14:53:19,566 INFO [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:19,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/.tmp/info/17cb3ed6bca44dadbcba67f9e13fe4b1 is 1080, key is row0008/info:/1731595986322/Put/seqid=0 2024-11-14T14:53:19,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741840_1016 (size=12509) 2024-11-14T14:53:19,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741840_1016 (size=12509) 2024-11-14T14:53:19,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/.tmp/info/17cb3ed6bca44dadbcba67f9e13fe4b1 2024-11-14T14:53:19,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/.tmp/info/17cb3ed6bca44dadbcba67f9e13fe4b1 as hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/17cb3ed6bca44dadbcba67f9e13fe4b1 2024-11-14T14:53:19,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/17cb3ed6bca44dadbcba67f9e13fe4b1, entries=7, sequenceid=21, filesize=12.2 K 2024-11-14T14:53:19,803 INFO [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:19,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 92807f1e92a275f371708d055eca2bfe in 439ms, sequenceid=21, compaction requested=false 2024-11-14T14:53:19,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92807f1e92a275f371708d055eca2bfe: 2024-11-14T14:53:19,804 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K 2024-11-14T14:53:19,804 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:53:19,805 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/e8a0dfe627684057a735bfb2dd37efa2 because midkey is the same as first or last row 2024-11-14T14:53:21,569 INFO [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:22,403 INFO [master/da1c6afcd1f9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-14T14:53:22,403 INFO [master/da1c6afcd1f9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-14T14:53:23,774 INFO [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:23,775 WARN [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:23,777 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog da1c6afcd1f9%2C33427%2C1731595959873:(num 1731595992334) roll requested 2024-11-14T14:53:23,777 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C33427%2C1731595959873.1731596003777 2024-11-14T14:53:23,986 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:23,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:23,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:23,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:23,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:23,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:23,987 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731595992334 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731596003777 2024-11-14T14:53:23,988 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43375:43375),(127.0.0.1/127.0.0.1:35365:35365)] 2024-11-14T14:53:23,988 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731595992334 is not closed yet, will try archiving it next time 2024-11-14T14:53:23,989 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731595972240 to hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/oldWALs/da1c6afcd1f9%2C33427%2C1731595959873.1731595972240 2024-11-14T14:53:23,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741839_1015 (size=7739) 2024-11-14T14:53:23,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741839_1015 (size=7739) 2024-11-14T14:53:25,978 INFO [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:27,547 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 92807f1e92a275f371708d055eca2bfe, had cached 0 bytes from a total of 25018 2024-11-14T14:53:28,182 INFO [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:30,386 INFO [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:32,591 INFO [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:34,593 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T14:53:34,594 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C33427%2C1731595959873.1731596014593 2024-11-14T14:53:38,330 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T14:53:39,603 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:39,604 WARN [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:39,605 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog da1c6afcd1f9%2C33427%2C1731595959873:(num 1731596014593) roll requested 2024-11-14T14:53:39,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:39,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:39,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:39,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:39,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:39,606 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731596003777 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731596014593 2024-11-14T14:53:39,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741841_1017 (size=4753) 2024-11-14T14:53:39,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741841_1017 (size=4753) 2024-11-14T14:53:39,609 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43375:43375),(127.0.0.1/127.0.0.1:35365:35365)] 2024-11-14T14:53:39,609 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731596003777 is not closed yet, will try archiving it next time 2024-11-14T14:53:39,609 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C33427%2C1731595959873.1731596019609 2024-11-14T14:53:44,612 INFO [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:44,612 WARN [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33427 {}] regionserver.HRegion(8855): Flush requested on 92807f1e92a275f371708d055eca2bfe 2024-11-14T14:53:44,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92807f1e92a275f371708d055eca2bfe 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-14T14:53:44,621 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5006 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:44,621 WARN [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5006 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:46,614 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T14:53:49,615 INFO [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:49,615 WARN [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:49,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:49,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:49,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:49,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:49,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:49,616 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731596014593 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731596019609 2024-11-14T14:53:49,618 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43375:43375),(127.0.0.1/127.0.0.1:35365:35365)] 2024-11-14T14:53:49,618 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731596014593 is not closed yet, will try archiving it next time 2024-11-14T14:53:49,618 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog da1c6afcd1f9%2C33427%2C1731595959873:(num 1731596019609) roll requested 2024-11-14T14:53:49,618 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C33427%2C1731595959873.1731596029618 2024-11-14T14:53:49,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741842_1018 (size=1569) 2024-11-14T14:53:49,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741842_1018 (size=1569) 2024-11-14T14:53:49,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/.tmp/info/9b69303f3cac4494adaca46d3eb43b5f is 1080, key is row0015/info:/1731596001367/Put/seqid=0 2024-11-14T14:53:49,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741844_1020 (size=12509) 2024-11-14T14:53:49,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741844_1020 (size=12509) 2024-11-14T14:53:49,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/.tmp/info/9b69303f3cac4494adaca46d3eb43b5f 2024-11-14T14:53:49,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/.tmp/info/9b69303f3cac4494adaca46d3eb43b5f as hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/9b69303f3cac4494adaca46d3eb43b5f 2024-11-14T14:53:49,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/9b69303f3cac4494adaca46d3eb43b5f, entries=7, sequenceid=31, filesize=12.2 K 2024-11-14T14:53:54,630 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:54,630 WARN [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:54,650 INFO [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:54,651 WARN [FSHLog-0-hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a-prefix:da1c6afcd1f9,33427,1731595959873 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5000 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:46015,DS-45f05b47-c238-41c5-8883-f6522d93c5d1,DISK], DatanodeInfoWithStorage[127.0.0.1:43709,DS-91fa873e-0cc6-49ec-952a-034e227e82b8,DISK]] 2024-11-14T14:53:54,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 92807f1e92a275f371708d055eca2bfe in 10037ms, sequenceid=31, compaction requested=true 2024-11-14T14:53:54,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:54,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92807f1e92a275f371708d055eca2bfe: 2024-11-14T14:53:54,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:54,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:54,651 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K 2024-11-14T14:53:54,651 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:53:54,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:54,651 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/e8a0dfe627684057a735bfb2dd37efa2 because midkey is the same as first or last row 2024-11-14T14:53:54,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:54,652 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731596019609 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731596029618 2024-11-14T14:53:54,653 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43375:43375),(127.0.0.1/127.0.0.1:35365:35365)] 2024-11-14T14:53:54,653 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731596019609 is not closed yet, will try archiving it next time 2024-11-14T14:53:54,653 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731595992334 to hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/oldWALs/da1c6afcd1f9%2C33427%2C1731595959873.1731595992334 2024-11-14T14:53:54,653 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog da1c6afcd1f9%2C33427%2C1731595959873:(num 1731596029618) roll requested 2024-11-14T14:53:54,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 92807f1e92a275f371708d055eca2bfe:info, priority=-2147483648, current under compaction store size is 1 2024-11-14T14:53:54,653 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C33427%2C1731595959873.1731596034653 2024-11-14T14:53:54,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741843_1019 (size=438) 2024-11-14T14:53:54,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741843_1019 (size=438) 2024-11-14T14:53:54,656 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731596003777 to hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/oldWALs/da1c6afcd1f9%2C33427%2C1731595959873.1731596003777 2024-11-14T14:53:54,656 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T14:53:54,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:53:54,657 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731596014593 to hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/oldWALs/da1c6afcd1f9%2C33427%2C1731595959873.1731596014593 2024-11-14T14:53:54,659 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731596019609 to hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/oldWALs/da1c6afcd1f9%2C33427%2C1731595959873.1731596019609 2024-11-14T14:53:54,659 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T14:53:54,661 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] regionserver.HStore(1541): 92807f1e92a275f371708d055eca2bfe/info is initiating minor compaction (all files) 2024-11-14T14:53:54,662 INFO [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 92807f1e92a275f371708d055eca2bfe/info in TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe. 2024-11-14T14:53:54,663 INFO [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/e8a0dfe627684057a735bfb2dd37efa2, hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/17cb3ed6bca44dadbcba67f9e13fe4b1, hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/9b69303f3cac4494adaca46d3eb43b5f] into tmpdir=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/.tmp, totalSize=36.6 K 2024-11-14T14:53:54,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:54,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:54,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:54,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:54,664 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8a0dfe627684057a735bfb2dd37efa2, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731595972288 2024-11-14T14:53:54,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:54,664 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731596029618 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731596034653 2024-11-14T14:53:54,665 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17cb3ed6bca44dadbcba67f9e13fe4b1, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1731595986322 2024-11-14T14:53:54,666 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b69303f3cac4494adaca46d3eb43b5f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1731596001367 2024-11-14T14:53:54,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741845_1021 (size=93) 2024-11-14T14:53:54,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741845_1021 (size=93) 2024-11-14T14:53:54,668 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731596029618 to hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/oldWALs/da1c6afcd1f9%2C33427%2C1731595959873.1731596029618 2024-11-14T14:53:54,674 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35365:35365),(127.0.0.1/127.0.0.1:43375:43375)] 2024-11-14T14:53:54,675 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C33427%2C1731595959873.1731596034674 2024-11-14T14:53:54,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:54,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:54,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:54,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:54,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:53:54,689 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731596034653 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/WALs/da1c6afcd1f9,33427,1731595959873/da1c6afcd1f9%2C33427%2C1731595959873.1731596034674 2024-11-14T14:53:54,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741846_1022 (size=1258) 2024-11-14T14:53:54,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741846_1022 (size=1258) 2024-11-14T14:53:54,693 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43375:43375),(127.0.0.1/127.0.0.1:35365:35365)] 2024-11-14T14:53:54,700 INFO [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 92807f1e92a275f371708d055eca2bfe#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T14:53:54,701 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/.tmp/info/09ed26dca70f4ea1842a5390788a54f9 is 1080, key is row0001/info:/1731595972288/Put/seqid=0 2024-11-14T14:53:54,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741848_1024 (size=27710) 2024-11-14T14:53:54,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741848_1024 (size=27710) 2024-11-14T14:53:54,719 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/.tmp/info/09ed26dca70f4ea1842a5390788a54f9 as hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/09ed26dca70f4ea1842a5390788a54f9 2024-11-14T14:53:54,735 INFO [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 92807f1e92a275f371708d055eca2bfe/info of 92807f1e92a275f371708d055eca2bfe into 09ed26dca70f4ea1842a5390788a54f9(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T14:53:54,736 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 92807f1e92a275f371708d055eca2bfe: 2024-11-14T14:53:54,738 INFO [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe., storeName=92807f1e92a275f371708d055eca2bfe/info, priority=13, startTime=1731596034653; duration=0sec 2024-11-14T14:53:54,738 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-11-14T14:53:54,738 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:53:54,738 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/09ed26dca70f4ea1842a5390788a54f9 because midkey is the same as first or last row 2024-11-14T14:53:54,739 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-11-14T14:53:54,739 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:53:54,739 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/09ed26dca70f4ea1842a5390788a54f9 because midkey is the same as first or last row 2024-11-14T14:53:54,739 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-11-14T14:53:54,739 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:53:54,739 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/09ed26dca70f4ea1842a5390788a54f9 because midkey is the same as first or last row 2024-11-14T14:53:54,739 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:53:54,739 DEBUG [RS:0;da1c6afcd1f9:33427-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 92807f1e92a275f371708d055eca2bfe:info 2024-11-14T14:54:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33427 {}] regionserver.HRegion(8855): Flush requested on 92807f1e92a275f371708d055eca2bfe 2024-11-14T14:54:06,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 92807f1e92a275f371708d055eca2bfe 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-14T14:54:06,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/.tmp/info/f8f331a451df4d07be10443a203a46bb is 1080, key is row0022/info:/1731596034676/Put/seqid=0 2024-11-14T14:54:06,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741849_1025 (size=12509) 2024-11-14T14:54:06,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741849_1025 (size=12509) 2024-11-14T14:54:06,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/.tmp/info/f8f331a451df4d07be10443a203a46bb 2024-11-14T14:54:06,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/.tmp/info/f8f331a451df4d07be10443a203a46bb as hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/f8f331a451df4d07be10443a203a46bb 2024-11-14T14:54:06,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/f8f331a451df4d07be10443a203a46bb, entries=7, sequenceid=42, filesize=12.2 K 2024-11-14T14:54:06,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 92807f1e92a275f371708d055eca2bfe in 42ms, sequenceid=42, compaction requested=false 2024-11-14T14:54:06,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 92807f1e92a275f371708d055eca2bfe: 2024-11-14T14:54:06,744 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K 2024-11-14T14:54:06,744 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:54:06,744 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/09ed26dca70f4ea1842a5390788a54f9 because midkey is the same as first or last row 2024-11-14T14:54:08,330 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T14:54:12,548 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 92807f1e92a275f371708d055eca2bfe, had cached 0 bytes from a total of 40219 2024-11-14T14:54:14,713 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T14:54:14,714 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:54:14,714 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:54:14,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:54:14,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:54:14,720 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T14:54:14,720 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T14:54:14,720 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1441288993, stopped=false 2024-11-14T14:54:14,721 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=da1c6afcd1f9,40347,1731595959216 2024-11-14T14:54:14,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:54:14,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:54:14,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:14,723 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:54:14,724 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:54:14,724 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:54:14,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:54:14,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:14,725 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'da1c6afcd1f9,33427,1731595959873' ***** 2024-11-14T14:54:14,725 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T14:54:14,725 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:54:14,726 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T14:54:14,726 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T14:54:14,726 INFO [RS:0;da1c6afcd1f9:33427 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T14:54:14,726 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:54:14,726 INFO [RS:0;da1c6afcd1f9:33427 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T14:54:14,727 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.HRegionServer(3091): Received CLOSE for 92807f1e92a275f371708d055eca2bfe 2024-11-14T14:54:14,727 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.HRegionServer(959): stopping server da1c6afcd1f9,33427,1731595959873 2024-11-14T14:54:14,727 INFO [RS:0;da1c6afcd1f9:33427 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:54:14,728 INFO [RS:0;da1c6afcd1f9:33427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;da1c6afcd1f9:33427. 2024-11-14T14:54:14,728 DEBUG [RS:0;da1c6afcd1f9:33427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:54:14,728 DEBUG [RS:0;da1c6afcd1f9:33427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:54:14,728 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 92807f1e92a275f371708d055eca2bfe, disabling compactions & flushes 2024-11-14T14:54:14,728 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe. 2024-11-14T14:54:14,728 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe. 2024-11-14T14:54:14,728 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe. after waiting 0 ms 2024-11-14T14:54:14,728 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe. 2024-11-14T14:54:14,728 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T14:54:14,728 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T14:54:14,728 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T14:54:14,728 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 92807f1e92a275f371708d055eca2bfe 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB 2024-11-14T14:54:14,728 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T14:54:14,729 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-14T14:54:14,729 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:54:14,729 DEBUG [RS:0;da1c6afcd1f9:33427 {}] regionserver.HRegionServer(1325): Online Regions={92807f1e92a275f371708d055eca2bfe=TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe., 1588230740=hbase:meta,,1.1588230740} 2024-11-14T14:54:14,729 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:54:14,729 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:54:14,729 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:54:14,729 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:54:14,730 DEBUG [RS:0;da1c6afcd1f9:33427 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 92807f1e92a275f371708d055eca2bfe 2024-11-14T14:54:14,730 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB 2024-11-14T14:54:14,737 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/.tmp/info/06fe613679fd40f7a33c32b51b95be18 is 1080, key is row0029/info:/1731596048704/Put/seqid=0 2024-11-14T14:54:14,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741850_1026 (size=8193) 2024-11-14T14:54:14,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741850_1026 (size=8193) 2024-11-14T14:54:14,745 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/.tmp/info/06fe613679fd40f7a33c32b51b95be18 2024-11-14T14:54:14,753 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740/.tmp/info/921ba720cb554afd89bc8f7d8ba49510 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe./info:regioninfo/1731595962574/Put/seqid=0 2024-11-14T14:54:14,755 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/.tmp/info/06fe613679fd40f7a33c32b51b95be18 as hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/06fe613679fd40f7a33c32b51b95be18 2024-11-14T14:54:14,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741851_1027 (size=7016) 2024-11-14T14:54:14,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741851_1027 (size=7016) 2024-11-14T14:54:14,760 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740/.tmp/info/921ba720cb554afd89bc8f7d8ba49510 2024-11-14T14:54:14,767 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/06fe613679fd40f7a33c32b51b95be18, entries=3, sequenceid=48, filesize=8.0 K 2024-11-14T14:54:14,769 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 92807f1e92a275f371708d055eca2bfe in 41ms, sequenceid=48, compaction requested=true 2024-11-14T14:54:14,770 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/e8a0dfe627684057a735bfb2dd37efa2, hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/17cb3ed6bca44dadbcba67f9e13fe4b1, hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/9b69303f3cac4494adaca46d3eb43b5f] to archive 2024-11-14T14:54:14,773 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T14:54:14,777 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/e8a0dfe627684057a735bfb2dd37efa2 to hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/archive/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/e8a0dfe627684057a735bfb2dd37efa2 2024-11-14T14:54:14,779 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/17cb3ed6bca44dadbcba67f9e13fe4b1 to hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/archive/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/17cb3ed6bca44dadbcba67f9e13fe4b1 2024-11-14T14:54:14,782 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/9b69303f3cac4494adaca46d3eb43b5f to hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/archive/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/info/9b69303f3cac4494adaca46d3eb43b5f 2024-11-14T14:54:14,791 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740/.tmp/ns/4291c3ca82fa4be1b12cabecd1ff6523 is 43, key is default/ns:d/1731595961873/Put/seqid=0 2024-11-14T14:54:14,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741852_1028 (size=5153) 2024-11-14T14:54:14,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741852_1028 (size=5153) 2024-11-14T14:54:14,801 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740/.tmp/ns/4291c3ca82fa4be1b12cabecd1ff6523 2024-11-14T14:54:14,796 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=da1c6afcd1f9:40347 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 16 more 2024-11-14T14:54:14,802 WARN [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [e8a0dfe627684057a735bfb2dd37efa2=12509, 17cb3ed6bca44dadbcba67f9e13fe4b1=12509, 9b69303f3cac4494adaca46d3eb43b5f=12509] 2024-11-14T14:54:14,809 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/default/TestLogRolling-testSlowSyncLogRolling/92807f1e92a275f371708d055eca2bfe/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1 2024-11-14T14:54:14,812 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe. 2024-11-14T14:54:14,812 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 92807f1e92a275f371708d055eca2bfe: Waiting for close lock at 1731596054727Running coprocessor pre-close hooks at 1731596054728 (+1 ms)Disabling compacts and flushes for region at 1731596054728Disabling writes for close at 1731596054728Obtaining lock to block concurrent updates at 1731596054728Preparing flush snapshotting stores in 92807f1e92a275f371708d055eca2bfe at 1731596054728Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1731596054729 (+1 ms)Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe. at 1731596054730 (+1 ms)Flushing 92807f1e92a275f371708d055eca2bfe/info: creating writer at 1731596054730Flushing 92807f1e92a275f371708d055eca2bfe/info: appending metadata at 1731596054736 (+6 ms)Flushing 92807f1e92a275f371708d055eca2bfe/info: closing flushed file at 1731596054736Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@470e9d90: reopening flushed file at 1731596054753 (+17 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 92807f1e92a275f371708d055eca2bfe in 41ms, sequenceid=48, compaction requested=true at 1731596054769 (+16 ms)Writing region close event to WAL at 1731596054803 (+34 ms)Running coprocessor post-close hooks at 1731596054810 (+7 ms)Closed at 1731596054812 (+2 ms) 2024-11-14T14:54:14,813 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1731595962101.92807f1e92a275f371708d055eca2bfe. 2024-11-14T14:54:14,826 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740/.tmp/table/841a64fa82094d828c2ccbe5cf96e6b6 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1731595962591/Put/seqid=0 2024-11-14T14:54:14,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741853_1029 (size=5396) 2024-11-14T14:54:14,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741853_1029 (size=5396) 2024-11-14T14:54:14,836 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740/.tmp/table/841a64fa82094d828c2ccbe5cf96e6b6 2024-11-14T14:54:14,849 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740/.tmp/info/921ba720cb554afd89bc8f7d8ba49510 as hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740/info/921ba720cb554afd89bc8f7d8ba49510 2024-11-14T14:54:14,860 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740/info/921ba720cb554afd89bc8f7d8ba49510, entries=10, sequenceid=11, filesize=6.9 K 2024-11-14T14:54:14,862 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740/.tmp/ns/4291c3ca82fa4be1b12cabecd1ff6523 as hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740/ns/4291c3ca82fa4be1b12cabecd1ff6523 2024-11-14T14:54:14,874 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740/ns/4291c3ca82fa4be1b12cabecd1ff6523, entries=2, sequenceid=11, filesize=5.0 K 2024-11-14T14:54:14,883 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740/.tmp/table/841a64fa82094d828c2ccbe5cf96e6b6 as hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740/table/841a64fa82094d828c2ccbe5cf96e6b6 2024-11-14T14:54:14,898 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740/table/841a64fa82094d828c2ccbe5cf96e6b6, entries=2, sequenceid=11, filesize=5.3 K 2024-11-14T14:54:14,901 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 170ms, sequenceid=11, compaction requested=false 2024-11-14T14:54:14,916 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-14T14:54:14,918 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:54:14,918 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:54:14,918 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731596054729Running coprocessor pre-close hooks at 1731596054729Disabling compacts and flushes for region at 1731596054729Disabling writes for close at 1731596054729Obtaining lock to block concurrent updates at 1731596054730 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731596054730Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1731596054730Flushing stores of hbase:meta,,1.1588230740 at 1731596054731 (+1 ms)Flushing 1588230740/info: creating writer at 1731596054732 (+1 ms)Flushing 1588230740/info: appending metadata at 1731596054752 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731596054752Flushing 1588230740/ns: creating writer at 1731596054768 (+16 ms)Flushing 1588230740/ns: appending metadata at 1731596054790 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1731596054790Flushing 1588230740/table: creating writer at 1731596054810 (+20 ms)Flushing 1588230740/table: appending metadata at 1731596054826 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731596054826Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e34582e: reopening flushed file at 1731596054848 (+22 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2acfdcc4: reopening flushed file at 1731596054861 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@104ce0ac: reopening flushed file at 1731596054874 (+13 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 170ms, sequenceid=11, compaction requested=false at 1731596054901 (+27 ms)Writing region close event to WAL at 1731596054910 (+9 ms)Running coprocessor post-close hooks at 1731596054917 (+7 ms)Closed at 1731596054918 (+1 ms) 2024-11-14T14:54:14,918 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T14:54:14,930 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.HRegionServer(976): stopping server da1c6afcd1f9,33427,1731595959873; all regions closed. 2024-11-14T14:54:14,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:14,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:14,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:14,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:14,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:14,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741834_1010 (size=3066) 2024-11-14T14:54:14,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741834_1010 (size=3066) 2024-11-14T14:54:14,940 DEBUG [RS:0;da1c6afcd1f9:33427 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/oldWALs 2024-11-14T14:54:14,940 INFO [RS:0;da1c6afcd1f9:33427 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog da1c6afcd1f9%2C33427%2C1731595959873.meta:.meta(num 1731595961675) 2024-11-14T14:54:14,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:14,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:14,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:14,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:14,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:14,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741847_1023 (size=12695) 2024-11-14T14:54:14,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741847_1023 (size=12695) 2024-11-14T14:54:14,949 DEBUG [RS:0;da1c6afcd1f9:33427 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/oldWALs 2024-11-14T14:54:14,949 INFO [RS:0;da1c6afcd1f9:33427 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog da1c6afcd1f9%2C33427%2C1731595959873:(num 1731596034674) 2024-11-14T14:54:14,949 DEBUG [RS:0;da1c6afcd1f9:33427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:54:14,949 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:54:14,949 INFO [RS:0;da1c6afcd1f9:33427 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:54:14,949 INFO [RS:0;da1c6afcd1f9:33427 {}] hbase.ChoreService(370): Chore service for: regionserver/da1c6afcd1f9:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T14:54:14,950 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:54:14,950 INFO [RS:0;da1c6afcd1f9:33427 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:54:14,951 INFO [RS:0;da1c6afcd1f9:33427 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33427 2024-11-14T14:54:14,956 INFO [RS:0;da1c6afcd1f9:33427 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:54:14,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:54:14,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/da1c6afcd1f9,33427,1731595959873 2024-11-14T14:54:14,959 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [da1c6afcd1f9,33427,1731595959873] 2024-11-14T14:54:14,961 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/da1c6afcd1f9,33427,1731595959873 already deleted, retry=false 2024-11-14T14:54:14,961 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; da1c6afcd1f9,33427,1731595959873 expired; onlineServers=0 2024-11-14T14:54:14,961 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'da1c6afcd1f9,40347,1731595959216' ***** 2024-11-14T14:54:14,961 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T14:54:14,962 INFO [M:0;da1c6afcd1f9:40347 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:54:14,962 INFO [M:0;da1c6afcd1f9:40347 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:54:14,962 DEBUG [M:0;da1c6afcd1f9:40347 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T14:54:14,962 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T14:54:14,962 DEBUG [M:0;da1c6afcd1f9:40347 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T14:54:14,962 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731595960919 {}] cleaner.HFileCleaner(306): Exit Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731595960919,5,FailOnTimeoutGroup] 2024-11-14T14:54:14,962 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731595960920 {}] cleaner.HFileCleaner(306): Exit Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731595960920,5,FailOnTimeoutGroup] 2024-11-14T14:54:14,962 INFO [M:0;da1c6afcd1f9:40347 {}] hbase.ChoreService(370): Chore service for: master/da1c6afcd1f9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T14:54:14,962 INFO [M:0;da1c6afcd1f9:40347 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:54:14,963 DEBUG [M:0;da1c6afcd1f9:40347 {}] master.HMaster(1795): Stopping service threads 2024-11-14T14:54:14,963 INFO [M:0;da1c6afcd1f9:40347 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T14:54:14,963 INFO [M:0;da1c6afcd1f9:40347 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:54:14,963 INFO [M:0;da1c6afcd1f9:40347 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:54:14,964 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T14:54:14,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T14:54:14,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:14,965 DEBUG [M:0;da1c6afcd1f9:40347 {}] zookeeper.ZKUtil(347): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T14:54:14,965 WARN [M:0;da1c6afcd1f9:40347 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T14:54:14,966 INFO [M:0;da1c6afcd1f9:40347 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/.lastflushedseqids 2024-11-14T14:54:14,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741854_1030 (size=130) 2024-11-14T14:54:14,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741854_1030 (size=130) 2024-11-14T14:54:14,996 INFO [M:0;da1c6afcd1f9:40347 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:54:14,996 INFO [M:0;da1c6afcd1f9:40347 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T14:54:14,996 DEBUG [M:0;da1c6afcd1f9:40347 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:54:14,997 INFO [M:0;da1c6afcd1f9:40347 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:54:14,997 DEBUG [M:0;da1c6afcd1f9:40347 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:54:14,997 DEBUG [M:0;da1c6afcd1f9:40347 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:54:14,997 DEBUG [M:0;da1c6afcd1f9:40347 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:54:14,997 INFO [M:0;da1c6afcd1f9:40347 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.02 KB heapSize=29.20 KB 2024-11-14T14:54:15,024 DEBUG [M:0;da1c6afcd1f9:40347 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c1328d27f3a4ab485da0c75ddd370c4 is 82, key is hbase:meta,,1/info:regioninfo/1731595961772/Put/seqid=0 2024-11-14T14:54:15,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741855_1031 (size=5672) 2024-11-14T14:54:15,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741855_1031 (size=5672) 2024-11-14T14:54:15,035 INFO [M:0;da1c6afcd1f9:40347 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c1328d27f3a4ab485da0c75ddd370c4 2024-11-14T14:54:15,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:54:15,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33427-0x101a744eddd0001, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:54:15,060 INFO [RS:0;da1c6afcd1f9:33427 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:54:15,060 INFO [RS:0;da1c6afcd1f9:33427 {}] regionserver.HRegionServer(1031): Exiting; stopping=da1c6afcd1f9,33427,1731595959873; zookeeper connection closed. 2024-11-14T14:54:15,061 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@cb00d9e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@cb00d9e 2024-11-14T14:54:15,062 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T14:54:15,064 DEBUG [M:0;da1c6afcd1f9:40347 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/15d5afbb766344249abb9887d483fffe is 766, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731595962599/Put/seqid=0 2024-11-14T14:54:15,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741856_1032 (size=6247) 2024-11-14T14:54:15,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741856_1032 (size=6247) 2024-11-14T14:54:15,124 INFO [regionserver/da1c6afcd1f9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:54:15,486 INFO [M:0;da1c6afcd1f9:40347 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/15d5afbb766344249abb9887d483fffe 2024-11-14T14:54:15,494 INFO [M:0;da1c6afcd1f9:40347 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 15d5afbb766344249abb9887d483fffe 2024-11-14T14:54:15,517 DEBUG [M:0;da1c6afcd1f9:40347 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/43f53cdac5c943f1a90f3837811e613e is 69, key is da1c6afcd1f9,33427,1731595959873/rs:state/1731595961032/Put/seqid=0 2024-11-14T14:54:15,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741857_1033 (size=5156) 2024-11-14T14:54:15,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741857_1033 (size=5156) 2024-11-14T14:54:15,527 INFO [M:0;da1c6afcd1f9:40347 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/43f53cdac5c943f1a90f3837811e613e 2024-11-14T14:54:15,555 DEBUG [M:0;da1c6afcd1f9:40347 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ef1c9c0b870a4cf183af4a69eb325d1d is 52, key is load_balancer_on/state:d/1731595962075/Put/seqid=0 2024-11-14T14:54:15,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741858_1034 (size=5056) 2024-11-14T14:54:15,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741858_1034 (size=5056) 2024-11-14T14:54:15,566 INFO [M:0;da1c6afcd1f9:40347 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ef1c9c0b870a4cf183af4a69eb325d1d 2024-11-14T14:54:15,575 DEBUG [M:0;da1c6afcd1f9:40347 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c1328d27f3a4ab485da0c75ddd370c4 as hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2c1328d27f3a4ab485da0c75ddd370c4 2024-11-14T14:54:15,582 INFO [M:0;da1c6afcd1f9:40347 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2c1328d27f3a4ab485da0c75ddd370c4, entries=8, sequenceid=59, filesize=5.5 K 2024-11-14T14:54:15,584 DEBUG [M:0;da1c6afcd1f9:40347 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/15d5afbb766344249abb9887d483fffe as hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/15d5afbb766344249abb9887d483fffe 2024-11-14T14:54:15,592 INFO [M:0;da1c6afcd1f9:40347 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 15d5afbb766344249abb9887d483fffe 2024-11-14T14:54:15,593 INFO [M:0;da1c6afcd1f9:40347 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/15d5afbb766344249abb9887d483fffe, entries=6, sequenceid=59, filesize=6.1 K 2024-11-14T14:54:15,594 DEBUG [M:0;da1c6afcd1f9:40347 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/43f53cdac5c943f1a90f3837811e613e as hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/43f53cdac5c943f1a90f3837811e613e 2024-11-14T14:54:15,603 INFO [M:0;da1c6afcd1f9:40347 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/43f53cdac5c943f1a90f3837811e613e, entries=1, sequenceid=59, filesize=5.0 K 2024-11-14T14:54:15,604 DEBUG [M:0;da1c6afcd1f9:40347 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ef1c9c0b870a4cf183af4a69eb325d1d as hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ef1c9c0b870a4cf183af4a69eb325d1d 2024-11-14T14:54:15,612 INFO [M:0;da1c6afcd1f9:40347 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ef1c9c0b870a4cf183af4a69eb325d1d, entries=1, sequenceid=59, filesize=4.9 K 2024-11-14T14:54:15,614 INFO [M:0;da1c6afcd1f9:40347 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 617ms, sequenceid=59, compaction requested=false 2024-11-14T14:54:15,618 INFO [M:0;da1c6afcd1f9:40347 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:54:15,618 DEBUG [M:0;da1c6afcd1f9:40347 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731596054996Disabling compacts and flushes for region at 1731596054996Disabling writes for close at 1731596054997 (+1 ms)Obtaining lock to block concurrent updates at 1731596054997Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731596054997Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23576, getHeapSize=29840, getOffHeapSize=0, getCellsCount=70 at 1731596054998 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731596054999 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731596054999Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731596055023 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731596055023Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731596055043 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731596055063 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731596055064 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731596055494 (+430 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731596055517 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731596055517Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731596055536 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731596055554 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731596055554Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39e465b9: reopening flushed file at 1731596055574 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dfb4e35: reopening flushed file at 1731596055583 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35cddff5: reopening flushed file at 1731596055593 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@626f7866: reopening flushed file at 1731596055603 (+10 ms)Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 617ms, sequenceid=59, compaction requested=false at 1731596055614 (+11 ms)Writing region close event to WAL at 1731596055618 (+4 ms)Closed at 1731596055618 2024-11-14T14:54:15,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:15,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:15,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:15,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:15,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:15,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43709 is added to blk_1073741830_1006 (size=27973) 2024-11-14T14:54:15,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46015 is added to blk_1073741830_1006 (size=27973) 2024-11-14T14:54:15,624 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:54:15,624 INFO [M:0;da1c6afcd1f9:40347 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T14:54:15,624 INFO [M:0;da1c6afcd1f9:40347 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40347 2024-11-14T14:54:15,625 INFO [M:0;da1c6afcd1f9:40347 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:54:15,727 INFO [M:0;da1c6afcd1f9:40347 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:54:15,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:54:15,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40347-0x101a744eddd0000, quorum=127.0.0.1:64763, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:54:15,732 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:54:15,735 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:54:15,736 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:54:15,736 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:54:15,736 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/hadoop.log.dir/,STOPPED} 2024-11-14T14:54:15,739 WARN [BP-494856361-172.17.0.2-1731595956337 heartbeating to localhost/127.0.0.1:37399 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:54:15,739 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:54:15,740 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:54:15,740 WARN [BP-494856361-172.17.0.2-1731595956337 heartbeating to localhost/127.0.0.1:37399 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-494856361-172.17.0.2-1731595956337 (Datanode Uuid d6da9506-e7e9-4967-9b25-916232d8b6f9) service to localhost/127.0.0.1:37399 2024-11-14T14:54:15,741 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/cluster_2aef49a5-0640-82c6-34f8-67754afcba77/data/data3/current/BP-494856361-172.17.0.2-1731595956337 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:54:15,741 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/cluster_2aef49a5-0640-82c6-34f8-67754afcba77/data/data4/current/BP-494856361-172.17.0.2-1731595956337 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:54:15,742 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:54:15,744 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:54:15,744 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:54:15,744 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:54:15,744 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:54:15,744 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/hadoop.log.dir/,STOPPED} 2024-11-14T14:54:15,746 WARN [BP-494856361-172.17.0.2-1731595956337 heartbeating to localhost/127.0.0.1:37399 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:54:15,746 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:54:15,746 WARN [BP-494856361-172.17.0.2-1731595956337 heartbeating to localhost/127.0.0.1:37399 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-494856361-172.17.0.2-1731595956337 (Datanode Uuid 612b49a2-9a4b-47b8-a208-807e10ffe37f) service to localhost/127.0.0.1:37399 2024-11-14T14:54:15,746 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:54:15,747 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/cluster_2aef49a5-0640-82c6-34f8-67754afcba77/data/data1/current/BP-494856361-172.17.0.2-1731595956337 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:54:15,747 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/cluster_2aef49a5-0640-82c6-34f8-67754afcba77/data/data2/current/BP-494856361-172.17.0.2-1731595956337 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:54:15,748 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:54:15,760 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T14:54:15,761 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:54:15,761 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:54:15,761 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:54:15,761 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/hadoop.log.dir/,STOPPED} 2024-11-14T14:54:15,770 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T14:54:15,816 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-14T14:54:15,830 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=80 (was 12) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37399 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37399 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: master/da1c6afcd1f9:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37399 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@6200af4c java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:37399 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37399 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins@localhost:37399 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/da1c6afcd1f9:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool-2-worker-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37399 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37399 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/da1c6afcd1f9:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=405 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=124 (was 161), ProcessCount=11 (was 11), AvailableMemoryMB=9069 (was 8591) - AvailableMemoryMB LEAK? - 2024-11-14T14:54:15,840 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=81, OpenFileDescriptor=405, MaxFileDescriptor=1048576, SystemLoadAverage=124, ProcessCount=11, AvailableMemoryMB=9067 2024-11-14T14:54:15,841 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T14:54:15,841 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/hadoop.log.dir so I do NOT create it in target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a 2024-11-14T14:54:15,841 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2adba570-b446-3f56-be83-d99d717f848d/hadoop.tmp.dir so I do NOT create it in target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a 2024-11-14T14:54:15,841 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/cluster_6fc3c328-0a8c-0ecb-19f2-2869cdda52d1, deleteOnExit=true 2024-11-14T14:54:15,841 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T14:54:15,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/test.cache.data in system properties and HBase conf 2024-11-14T14:54:15,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T14:54:15,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/hadoop.log.dir in system properties and HBase conf 2024-11-14T14:54:15,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T14:54:15,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T14:54:15,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T14:54:15,843 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T14:54:15,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T14:54:15,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T14:54:15,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T14:54:15,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T14:54:15,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T14:54:15,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T14:54:15,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T14:54:15,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T14:54:15,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T14:54:15,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/nfs.dump.dir in system properties and HBase conf 2024-11-14T14:54:15,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/java.io.tmpdir in system properties and HBase conf 2024-11-14T14:54:15,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T14:54:15,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T14:54:15,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T14:54:15,868 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-14T14:54:15,959 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:54:15,967 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:54:15,970 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:54:15,970 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:54:15,970 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T14:54:15,971 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:54:15,972 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a3c3ceb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:54:15,973 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a69944b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:54:16,117 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3235d5ba{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/java.io.tmpdir/jetty-localhost-38045-hadoop-hdfs-3_4_1-tests_jar-_-any-5887077208750021022/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T14:54:16,118 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@347af0d{HTTP/1.1, (http/1.1)}{localhost:38045} 2024-11-14T14:54:16,118 INFO [Time-limited test {}] server.Server(415): Started @101740ms 2024-11-14T14:54:16,137 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-14T14:54:16,238 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:54:16,243 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:54:16,244 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:54:16,244 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:54:16,244 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T14:54:16,245 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@607b9bc6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:54:16,245 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@670e4080{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:54:16,373 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@61e52b83{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/java.io.tmpdir/jetty-localhost-37357-hadoop-hdfs-3_4_1-tests_jar-_-any-11475423339434157184/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:54:16,374 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@29a18ee0{HTTP/1.1, (http/1.1)}{localhost:37357} 2024-11-14T14:54:16,374 INFO [Time-limited test {}] server.Server(415): Started @101996ms 2024-11-14T14:54:16,377 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:54:16,419 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:54:16,423 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:54:16,424 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:54:16,424 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:54:16,424 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T14:54:16,425 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d944f53{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:54:16,425 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@18f27499{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:54:16,487 WARN [Thread-438 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/cluster_6fc3c328-0a8c-0ecb-19f2-2869cdda52d1/data/data1/current/BP-199397938-172.17.0.2-1731596055892/current, will proceed with Du for space computation calculation, 2024-11-14T14:54:16,487 WARN [Thread-439 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/cluster_6fc3c328-0a8c-0ecb-19f2-2869cdda52d1/data/data2/current/BP-199397938-172.17.0.2-1731596055892/current, will proceed with Du for space computation calculation, 2024-11-14T14:54:16,514 WARN [Thread-417 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:54:16,517 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4ad25638a71facab with lease ID 0x82c538a4aa8c205c: Processing first storage report for DS-ca0fd41a-c4f4-45c9-a433-d2d94f0105cc from datanode DatanodeRegistration(127.0.0.1:41639, datanodeUuid=5b21d4fd-8a38-4cb3-90e2-4bf2685a8269, infoPort=37615, infoSecurePort=0, ipcPort=36215, storageInfo=lv=-57;cid=testClusterID;nsid=749879802;c=1731596055892) 2024-11-14T14:54:16,518 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4ad25638a71facab with lease ID 0x82c538a4aa8c205c: from storage DS-ca0fd41a-c4f4-45c9-a433-d2d94f0105cc node DatanodeRegistration(127.0.0.1:41639, datanodeUuid=5b21d4fd-8a38-4cb3-90e2-4bf2685a8269, infoPort=37615, infoSecurePort=0, ipcPort=36215, storageInfo=lv=-57;cid=testClusterID;nsid=749879802;c=1731596055892), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:54:16,518 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4ad25638a71facab with lease ID 0x82c538a4aa8c205c: Processing first storage report for DS-8d987773-a53b-4b2e-96b1-9ca3d863d7c6 from datanode DatanodeRegistration(127.0.0.1:41639, datanodeUuid=5b21d4fd-8a38-4cb3-90e2-4bf2685a8269, infoPort=37615, infoSecurePort=0, ipcPort=36215, storageInfo=lv=-57;cid=testClusterID;nsid=749879802;c=1731596055892) 2024-11-14T14:54:16,518 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4ad25638a71facab with lease ID 0x82c538a4aa8c205c: from storage DS-8d987773-a53b-4b2e-96b1-9ca3d863d7c6 node DatanodeRegistration(127.0.0.1:41639, datanodeUuid=5b21d4fd-8a38-4cb3-90e2-4bf2685a8269, infoPort=37615, infoSecurePort=0, ipcPort=36215, storageInfo=lv=-57;cid=testClusterID;nsid=749879802;c=1731596055892), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:54:16,549 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cebd4b6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/java.io.tmpdir/jetty-localhost-40381-hadoop-hdfs-3_4_1-tests_jar-_-any-15913237086890407619/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:54:16,550 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1e6ff2f3{HTTP/1.1, (http/1.1)}{localhost:40381} 2024-11-14T14:54:16,550 INFO [Time-limited test {}] server.Server(415): Started @102171ms 2024-11-14T14:54:16,551 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:54:16,651 WARN [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/cluster_6fc3c328-0a8c-0ecb-19f2-2869cdda52d1/data/data3/current/BP-199397938-172.17.0.2-1731596055892/current, will proceed with Du for space computation calculation, 2024-11-14T14:54:16,651 WARN [Thread-465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/cluster_6fc3c328-0a8c-0ecb-19f2-2869cdda52d1/data/data4/current/BP-199397938-172.17.0.2-1731596055892/current, will proceed with Du for space computation calculation, 2024-11-14T14:54:16,676 WARN [Thread-453 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:54:16,679 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x29d5660074d5f108 with lease ID 0x82c538a4aa8c205d: Processing first storage report for DS-5a080a96-6299-4e19-a93b-0a94309e4f64 from datanode DatanodeRegistration(127.0.0.1:34537, datanodeUuid=9b52ebbe-807e-4c96-aabc-f6450c040781, infoPort=33731, infoSecurePort=0, ipcPort=33293, storageInfo=lv=-57;cid=testClusterID;nsid=749879802;c=1731596055892) 2024-11-14T14:54:16,679 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x29d5660074d5f108 with lease ID 0x82c538a4aa8c205d: from storage DS-5a080a96-6299-4e19-a93b-0a94309e4f64 node DatanodeRegistration(127.0.0.1:34537, datanodeUuid=9b52ebbe-807e-4c96-aabc-f6450c040781, infoPort=33731, infoSecurePort=0, ipcPort=33293, storageInfo=lv=-57;cid=testClusterID;nsid=749879802;c=1731596055892), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T14:54:16,679 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x29d5660074d5f108 with lease ID 0x82c538a4aa8c205d: Processing first storage report for DS-0f55ae44-7595-4f33-b323-be9a655915c6 from datanode DatanodeRegistration(127.0.0.1:34537, datanodeUuid=9b52ebbe-807e-4c96-aabc-f6450c040781, infoPort=33731, infoSecurePort=0, ipcPort=33293, storageInfo=lv=-57;cid=testClusterID;nsid=749879802;c=1731596055892) 2024-11-14T14:54:16,679 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x29d5660074d5f108 with lease ID 0x82c538a4aa8c205d: from storage DS-0f55ae44-7595-4f33-b323-be9a655915c6 node DatanodeRegistration(127.0.0.1:34537, datanodeUuid=9b52ebbe-807e-4c96-aabc-f6450c040781, infoPort=33731, infoSecurePort=0, ipcPort=33293, storageInfo=lv=-57;cid=testClusterID;nsid=749879802;c=1731596055892), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:54:16,698 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a 2024-11-14T14:54:16,701 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/cluster_6fc3c328-0a8c-0ecb-19f2-2869cdda52d1/zookeeper_0, clientPort=52554, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/cluster_6fc3c328-0a8c-0ecb-19f2-2869cdda52d1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/cluster_6fc3c328-0a8c-0ecb-19f2-2869cdda52d1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T14:54:16,702 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52554 2024-11-14T14:54:16,703 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:54:16,704 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:54:16,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41639 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:54:16,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34537 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:54:16,717 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b with version=8 2024-11-14T14:54:16,717 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/hbase-staging 2024-11-14T14:54:16,719 INFO [Time-limited test {}] client.ConnectionUtils(128): master/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:54:16,720 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:54:16,720 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:54:16,720 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:54:16,720 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:54:16,720 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:54:16,720 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T14:54:16,720 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:54:16,721 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44379 2024-11-14T14:54:16,722 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44379 connecting to ZooKeeper ensemble=127.0.0.1:52554 2024-11-14T14:54:16,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:443790x0, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:54:16,729 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44379-0x101a7466de50000 connected 2024-11-14T14:54:16,749 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:54:16,751 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:54:16,753 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:54:16,753 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b, hbase.cluster.distributed=false 2024-11-14T14:54:16,755 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:54:16,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44379 2024-11-14T14:54:16,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44379 2024-11-14T14:54:16,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44379 2024-11-14T14:54:16,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44379 2024-11-14T14:54:16,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44379 2024-11-14T14:54:16,775 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:54:16,775 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:54:16,775 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:54:16,776 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:54:16,776 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:54:16,776 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:54:16,776 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T14:54:16,776 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:54:16,777 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40381 2024-11-14T14:54:16,778 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40381 connecting to ZooKeeper ensemble=127.0.0.1:52554 2024-11-14T14:54:16,779 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:54:16,781 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:54:16,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:403810x0, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:54:16,787 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40381-0x101a7466de50001 connected 2024-11-14T14:54:16,791 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:54:16,791 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T14:54:16,792 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T14:54:16,793 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T14:54:16,795 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:54:16,795 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40381 2024-11-14T14:54:16,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40381 2024-11-14T14:54:16,799 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40381 2024-11-14T14:54:16,800 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40381 2024-11-14T14:54:16,803 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40381 2024-11-14T14:54:16,819 DEBUG [M:0;da1c6afcd1f9:44379 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;da1c6afcd1f9:44379 2024-11-14T14:54:16,819 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/da1c6afcd1f9,44379,1731596056719 2024-11-14T14:54:16,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:54:16,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:54:16,822 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/da1c6afcd1f9,44379,1731596056719 2024-11-14T14:54:16,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T14:54:16,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:16,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:16,825 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T14:54:16,826 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/da1c6afcd1f9,44379,1731596056719 from backup master directory 2024-11-14T14:54:16,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/da1c6afcd1f9,44379,1731596056719 2024-11-14T14:54:16,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:54:16,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:54:16,827 WARN [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:54:16,827 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=da1c6afcd1f9,44379,1731596056719 2024-11-14T14:54:16,838 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/hbase.id] with ID: d7865910-2719-40de-8455-fd0b91146287 2024-11-14T14:54:16,838 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/.tmp/hbase.id 2024-11-14T14:54:16,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41639 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:54:16,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34537 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:54:16,847 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/.tmp/hbase.id]:[hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/hbase.id] 2024-11-14T14:54:16,865 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:54:16,865 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T14:54:16,867 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-14T14:54:16,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:16,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:16,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41639 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:54:16,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34537 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:54:16,880 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T14:54:16,881 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T14:54:16,882 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:54:16,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41639 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:54:16,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34537 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:54:16,895 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store 2024-11-14T14:54:16,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41639 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:54:16,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34537 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:54:16,916 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:54:16,917 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:54:16,917 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:54:16,917 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:54:16,917 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:54:16,917 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:54:16,917 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:54:16,918 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731596056916Disabling compacts and flushes for region at 1731596056916Disabling writes for close at 1731596056917 (+1 ms)Writing region close event to WAL at 1731596056917Closed at 1731596056917 2024-11-14T14:54:16,920 WARN [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/.initializing 2024-11-14T14:54:16,920 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/WALs/da1c6afcd1f9,44379,1731596056719 2024-11-14T14:54:16,924 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C44379%2C1731596056719, suffix=, logDir=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/WALs/da1c6afcd1f9,44379,1731596056719, archiveDir=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/oldWALs, maxLogs=10 2024-11-14T14:54:16,925 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C44379%2C1731596056719.1731596056924 2024-11-14T14:54:16,944 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/WALs/da1c6afcd1f9,44379,1731596056719/da1c6afcd1f9%2C44379%2C1731596056719.1731596056924 2024-11-14T14:54:16,948 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33731:33731),(127.0.0.1/127.0.0.1:37615:37615)] 2024-11-14T14:54:16,949 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:54:16,950 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:54:16,950 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:16,950 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:16,953 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:16,956 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T14:54:16,956 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:16,957 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:54:16,957 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:16,959 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T14:54:16,959 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:16,960 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:54:16,960 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:16,963 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T14:54:16,963 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:16,964 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:54:16,964 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:16,966 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T14:54:16,966 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:16,966 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:54:16,967 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:16,968 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:16,968 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:16,970 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:16,970 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:16,971 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T14:54:16,973 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:16,976 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:54:16,976 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=743377, jitterRate=-0.05474771559238434}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T14:54:16,978 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731596056950Initializing all the Stores at 1731596056952 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596056952Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596056953 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596056953Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596056953Cleaning up temporary data from old regions at 1731596056970 (+17 ms)Region opened successfully at 1731596056978 (+8 ms) 2024-11-14T14:54:16,978 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T14:54:16,983 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@194799fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:54:16,984 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T14:54:16,984 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T14:54:16,984 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T14:54:16,985 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:54:16,985 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:54:16,986 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-14T14:54:16,986 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T14:54:16,991 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T14:54:16,993 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T14:54:16,995 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T14:54:16,995 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T14:54:16,996 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T14:54:16,997 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T14:54:16,998 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T14:54:17,003 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T14:54:17,005 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T14:54:17,007 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T14:54:17,008 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T14:54:17,013 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T14:54:17,015 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T14:54:17,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:54:17,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:54:17,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:17,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:17,020 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=da1c6afcd1f9,44379,1731596056719, sessionid=0x101a7466de50000, setting cluster-up flag (Was=false) 2024-11-14T14:54:17,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:17,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:17,033 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T14:54:17,035 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da1c6afcd1f9,44379,1731596056719 2024-11-14T14:54:17,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:17,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:17,047 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T14:54:17,049 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da1c6afcd1f9,44379,1731596056719 2024-11-14T14:54:17,052 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T14:54:17,054 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:54:17,055 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T14:54:17,055 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T14:54:17,055 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: da1c6afcd1f9,44379,1731596056719 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T14:54:17,057 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:54:17,057 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:54:17,057 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:54:17,057 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:54:17,057 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/da1c6afcd1f9:0, corePoolSize=10, maxPoolSize=10 2024-11-14T14:54:17,057 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:17,058 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:54:17,058 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:17,059 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731596087059 2024-11-14T14:54:17,059 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T14:54:17,059 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T14:54:17,059 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T14:54:17,059 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T14:54:17,059 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T14:54:17,060 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T14:54:17,060 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,060 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:54:17,060 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T14:54:17,060 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T14:54:17,060 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T14:54:17,061 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T14:54:17,062 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:17,062 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T14:54:17,062 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T14:54:17,062 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T14:54:17,064 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731596057063,5,FailOnTimeoutGroup] 2024-11-14T14:54:17,070 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731596057065,5,FailOnTimeoutGroup] 2024-11-14T14:54:17,070 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,071 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T14:54:17,071 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,071 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41639 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:54:17,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34537 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:54:17,084 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T14:54:17,084 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b 2024-11-14T14:54:17,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34537 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:54:17,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41639 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:54:17,101 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:54:17,103 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:54:17,105 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:54:17,105 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:17,105 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.HRegionServer(746): ClusterId : d7865910-2719-40de-8455-fd0b91146287 2024-11-14T14:54:17,105 DEBUG [RS:0;da1c6afcd1f9:40381 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T14:54:17,106 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:54:17,106 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:54:17,109 DEBUG [RS:0;da1c6afcd1f9:40381 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T14:54:17,109 DEBUG [RS:0;da1c6afcd1f9:40381 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T14:54:17,112 DEBUG [RS:0;da1c6afcd1f9:40381 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T14:54:17,113 DEBUG [RS:0;da1c6afcd1f9:40381 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16afaf9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:54:17,117 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:54:17,117 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:17,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:54:17,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:54:17,121 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:54:17,121 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:17,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:54:17,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:54:17,124 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:54:17,124 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:17,125 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:54:17,125 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:54:17,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/data/hbase/meta/1588230740 2024-11-14T14:54:17,127 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/data/hbase/meta/1588230740 2024-11-14T14:54:17,128 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:54:17,129 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:54:17,129 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T14:54:17,131 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:54:17,141 DEBUG [RS:0;da1c6afcd1f9:40381 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;da1c6afcd1f9:40381 2024-11-14T14:54:17,141 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:54:17,141 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T14:54:17,141 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T14:54:17,141 DEBUG [RS:0;da1c6afcd1f9:40381 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T14:54:17,142 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=818309, jitterRate=0.04053477942943573}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T14:54:17,143 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.HRegionServer(2659): reportForDuty to master=da1c6afcd1f9,44379,1731596056719 with port=40381, startcode=1731596056775 2024-11-14T14:54:17,143 DEBUG [RS:0;da1c6afcd1f9:40381 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T14:54:17,143 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731596057101Initializing all the Stores at 1731596057102 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596057102Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596057103 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596057103Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596057103Cleaning up temporary data from old regions at 1731596057129 (+26 ms)Region opened successfully at 1731596057143 (+14 ms) 2024-11-14T14:54:17,144 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:54:17,144 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:54:17,144 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:54:17,144 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:54:17,144 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:54:17,150 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:54:17,150 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731596057144Disabling compacts and flushes for region at 1731596057144Disabling writes for close at 1731596057144Writing region close event to WAL at 1731596057150 (+6 ms)Closed at 1731596057150 2024-11-14T14:54:17,152 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:54:17,152 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T14:54:17,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T14:54:17,155 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:54:17,157 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T14:54:17,158 INFO [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55867, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T14:54:17,159 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44379 {}] master.ServerManager(363): Checking decommissioned status of RegionServer da1c6afcd1f9,40381,1731596056775 2024-11-14T14:54:17,159 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44379 {}] master.ServerManager(517): Registering regionserver=da1c6afcd1f9,40381,1731596056775 2024-11-14T14:54:17,161 DEBUG [RS:0;da1c6afcd1f9:40381 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b 2024-11-14T14:54:17,162 DEBUG [RS:0;da1c6afcd1f9:40381 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42069 2024-11-14T14:54:17,162 DEBUG [RS:0;da1c6afcd1f9:40381 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T14:54:17,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:54:17,165 DEBUG [RS:0;da1c6afcd1f9:40381 {}] zookeeper.ZKUtil(111): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/da1c6afcd1f9,40381,1731596056775 2024-11-14T14:54:17,165 WARN [RS:0;da1c6afcd1f9:40381 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:54:17,165 INFO [RS:0;da1c6afcd1f9:40381 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:54:17,165 DEBUG [RS:0;da1c6afcd1f9:40381 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/WALs/da1c6afcd1f9,40381,1731596056775 2024-11-14T14:54:17,168 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [da1c6afcd1f9,40381,1731596056775] 2024-11-14T14:54:17,177 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T14:54:17,184 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T14:54:17,184 INFO [RS:0;da1c6afcd1f9:40381 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T14:54:17,185 INFO [RS:0;da1c6afcd1f9:40381 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,185 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T14:54:17,186 INFO [RS:0;da1c6afcd1f9:40381 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T14:54:17,187 INFO [RS:0;da1c6afcd1f9:40381 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,187 DEBUG [RS:0;da1c6afcd1f9:40381 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:17,187 DEBUG [RS:0;da1c6afcd1f9:40381 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:17,187 DEBUG [RS:0;da1c6afcd1f9:40381 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:17,187 DEBUG [RS:0;da1c6afcd1f9:40381 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:17,187 DEBUG [RS:0;da1c6afcd1f9:40381 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:17,187 DEBUG [RS:0;da1c6afcd1f9:40381 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:54:17,187 DEBUG [RS:0;da1c6afcd1f9:40381 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:17,188 DEBUG [RS:0;da1c6afcd1f9:40381 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:17,188 DEBUG [RS:0;da1c6afcd1f9:40381 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:17,188 DEBUG [RS:0;da1c6afcd1f9:40381 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:17,188 DEBUG [RS:0;da1c6afcd1f9:40381 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:17,188 DEBUG [RS:0;da1c6afcd1f9:40381 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:17,188 DEBUG [RS:0;da1c6afcd1f9:40381 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:54:17,188 DEBUG [RS:0;da1c6afcd1f9:40381 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:54:17,189 INFO [RS:0;da1c6afcd1f9:40381 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,189 INFO [RS:0;da1c6afcd1f9:40381 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,189 INFO [RS:0;da1c6afcd1f9:40381 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,189 INFO [RS:0;da1c6afcd1f9:40381 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,189 INFO [RS:0;da1c6afcd1f9:40381 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,189 INFO [RS:0;da1c6afcd1f9:40381 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40381,1731596056775-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:54:17,215 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T14:54:17,215 INFO [RS:0;da1c6afcd1f9:40381 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40381,1731596056775-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,215 INFO [RS:0;da1c6afcd1f9:40381 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,215 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.Replication(171): da1c6afcd1f9,40381,1731596056775 started 2024-11-14T14:54:17,235 INFO [RS:0;da1c6afcd1f9:40381 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,235 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.HRegionServer(1482): Serving as da1c6afcd1f9,40381,1731596056775, RpcServer on da1c6afcd1f9/172.17.0.2:40381, sessionid=0x101a7466de50001 2024-11-14T14:54:17,236 DEBUG [RS:0;da1c6afcd1f9:40381 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T14:54:17,236 DEBUG [RS:0;da1c6afcd1f9:40381 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager da1c6afcd1f9,40381,1731596056775 2024-11-14T14:54:17,236 DEBUG [RS:0;da1c6afcd1f9:40381 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,40381,1731596056775' 2024-11-14T14:54:17,236 DEBUG [RS:0;da1c6afcd1f9:40381 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T14:54:17,237 DEBUG [RS:0;da1c6afcd1f9:40381 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T14:54:17,238 DEBUG [RS:0;da1c6afcd1f9:40381 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T14:54:17,238 DEBUG [RS:0;da1c6afcd1f9:40381 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T14:54:17,238 DEBUG [RS:0;da1c6afcd1f9:40381 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager da1c6afcd1f9,40381,1731596056775 2024-11-14T14:54:17,238 DEBUG [RS:0;da1c6afcd1f9:40381 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,40381,1731596056775' 2024-11-14T14:54:17,238 DEBUG [RS:0;da1c6afcd1f9:40381 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T14:54:17,239 DEBUG [RS:0;da1c6afcd1f9:40381 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T14:54:17,239 DEBUG [RS:0;da1c6afcd1f9:40381 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T14:54:17,239 INFO [RS:0;da1c6afcd1f9:40381 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T14:54:17,239 INFO [RS:0;da1c6afcd1f9:40381 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T14:54:17,308 WARN [da1c6afcd1f9:44379 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-14T14:54:17,342 INFO [RS:0;da1c6afcd1f9:40381 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C40381%2C1731596056775, suffix=, logDir=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/WALs/da1c6afcd1f9,40381,1731596056775, archiveDir=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/oldWALs, maxLogs=32 2024-11-14T14:54:17,344 INFO [RS:0;da1c6afcd1f9:40381 {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C40381%2C1731596056775.1731596057344 2024-11-14T14:54:17,355 INFO [RS:0;da1c6afcd1f9:40381 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/WALs/da1c6afcd1f9,40381,1731596056775/da1c6afcd1f9%2C40381%2C1731596056775.1731596057344 2024-11-14T14:54:17,361 DEBUG [RS:0;da1c6afcd1f9:40381 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33731:33731),(127.0.0.1/127.0.0.1:37615:37615)] 2024-11-14T14:54:17,558 DEBUG [da1c6afcd1f9:44379 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T14:54:17,559 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=da1c6afcd1f9,40381,1731596056775 2024-11-14T14:54:17,561 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40381,1731596056775, state=OPENING 2024-11-14T14:54:17,563 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T14:54:17,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:17,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:17,566 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:54:17,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40381,1731596056775}] 2024-11-14T14:54:17,566 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:54:17,566 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:54:17,720 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T14:54:17,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33575, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T14:54:17,738 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T14:54:17,738 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:54:17,741 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C40381%2C1731596056775.meta, suffix=.meta, logDir=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/WALs/da1c6afcd1f9,40381,1731596056775, archiveDir=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/oldWALs, maxLogs=32 2024-11-14T14:54:17,743 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C40381%2C1731596056775.meta.1731596057743.meta 2024-11-14T14:54:17,754 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/WALs/da1c6afcd1f9,40381,1731596056775/da1c6afcd1f9%2C40381%2C1731596056775.meta.1731596057743.meta 2024-11-14T14:54:17,758 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37615:37615),(127.0.0.1/127.0.0.1:33731:33731)] 2024-11-14T14:54:17,761 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:54:17,761 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T14:54:17,761 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T14:54:17,762 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T14:54:17,762 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T14:54:17,762 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:54:17,762 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T14:54:17,762 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T14:54:17,770 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:54:17,772 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:54:17,772 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:17,773 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:54:17,773 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:54:17,774 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:54:17,774 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:17,775 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:54:17,775 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:54:17,777 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:54:17,777 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:17,778 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:54:17,778 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:54:17,779 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:54:17,779 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:17,780 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:54:17,780 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:54:17,781 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/data/hbase/meta/1588230740 2024-11-14T14:54:17,783 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/data/hbase/meta/1588230740 2024-11-14T14:54:17,785 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:54:17,785 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:54:17,786 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T14:54:17,790 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:54:17,791 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=759433, jitterRate=-0.034332215785980225}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T14:54:17,792 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T14:54:17,793 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731596057762Writing region info on filesystem at 1731596057762Initializing all the Stores at 1731596057768 (+6 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596057768Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596057770 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596057770Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596057770Cleaning up temporary data from old regions at 1731596057785 (+15 ms)Running coprocessor post-open hooks at 1731596057792 (+7 ms)Region opened successfully at 1731596057793 (+1 ms) 2024-11-14T14:54:17,795 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731596057720 2024-11-14T14:54:17,799 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T14:54:17,799 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T14:54:17,800 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,40381,1731596056775 2024-11-14T14:54:17,802 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40381,1731596056775, state=OPEN 2024-11-14T14:54:17,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:54:17,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:54:17,807 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:54:17,807 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:54:17,807 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40381,1731596056775 2024-11-14T14:54:17,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T14:54:17,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40381,1731596056775 in 241 msec 2024-11-14T14:54:17,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T14:54:17,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 659 msec 2024-11-14T14:54:17,818 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:54:17,818 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T14:54:17,821 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:54:17,821 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,40381,1731596056775, seqNum=-1] 2024-11-14T14:54:17,821 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:54:17,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43185, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:54:17,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 778 msec 2024-11-14T14:54:17,834 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731596057834, completionTime=-1 2024-11-14T14:54:17,834 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T14:54:17,834 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:54:17,837 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-14T14:54:17,837 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731596117837 2024-11-14T14:54:17,837 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731596177837 2024-11-14T14:54:17,837 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-14T14:54:17,838 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,44379,1731596056719-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,838 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,44379,1731596056719-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,838 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,44379,1731596056719-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,838 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-da1c6afcd1f9:44379, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,838 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,839 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,841 DEBUG [master/da1c6afcd1f9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T14:54:17,845 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.018sec 2024-11-14T14:54:17,845 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T14:54:17,845 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T14:54:17,845 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T14:54:17,846 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T14:54:17,846 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T14:54:17,846 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,44379,1731596056719-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:54:17,846 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,44379,1731596056719-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T14:54:17,850 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T14:54:17,850 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T14:54:17,850 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,44379,1731596056719-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:17,910 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205bfd42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:54:17,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request da1c6afcd1f9,44379,-1 for getting cluster id 2024-11-14T14:54:17,911 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T14:54:17,930 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd7865910-2719-40de-8455-fd0b91146287' 2024-11-14T14:54:17,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T14:54:17,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d7865910-2719-40de-8455-fd0b91146287" 2024-11-14T14:54:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d3e7eb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:54:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [da1c6afcd1f9,44379,-1] 2024-11-14T14:54:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T14:54:17,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:54:17,949 INFO [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T14:54:17,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60dafc4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:54:17,952 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:54:17,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,40381,1731596056775, seqNum=-1] 2024-11-14T14:54:17,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:54:17,959 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:54:17,964 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=da1c6afcd1f9,44379,1731596056719 2024-11-14T14:54:17,965 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:54:17,969 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-14T14:54:17,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T14:54:17,969 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:54:17,969 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:54:17,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:54:17,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:54:17,970 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T14:54:17,970 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T14:54:17,970 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1676223287, stopped=false 2024-11-14T14:54:17,971 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=da1c6afcd1f9,44379,1731596056719 2024-11-14T14:54:17,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:54:17,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:54:17,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:17,973 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:17,973 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:54:17,974 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:54:17,974 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:54:17,976 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:54:17,976 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:54:17,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:54:17,976 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'da1c6afcd1f9,40381,1731596056775' ***** 2024-11-14T14:54:17,977 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T14:54:17,977 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T14:54:17,977 INFO [RS:0;da1c6afcd1f9:40381 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T14:54:17,977 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T14:54:17,977 INFO [RS:0;da1c6afcd1f9:40381 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T14:54:17,977 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.HRegionServer(959): stopping server da1c6afcd1f9,40381,1731596056775 2024-11-14T14:54:17,977 INFO [RS:0;da1c6afcd1f9:40381 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:54:17,977 INFO [RS:0;da1c6afcd1f9:40381 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;da1c6afcd1f9:40381. 2024-11-14T14:54:17,977 DEBUG [RS:0;da1c6afcd1f9:40381 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:54:17,977 DEBUG [RS:0;da1c6afcd1f9:40381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:54:17,977 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T14:54:17,977 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T14:54:17,977 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T14:54:17,978 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T14:54:17,980 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T14:54:17,980 DEBUG [RS:0;da1c6afcd1f9:40381 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-14T14:54:17,980 DEBUG [RS:0;da1c6afcd1f9:40381 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T14:54:17,980 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:54:17,980 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:54:17,980 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:54:17,980 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:54:17,980 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:54:17,981 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-11-14T14:54:18,010 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/data/hbase/meta/1588230740/.tmp/ns/209fd144d478484bba2e155e1543c221 is 43, key is default/ns:d/1731596057824/Put/seqid=0 2024-11-14T14:54:18,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34537 is added to blk_1073741835_1011 (size=5153) 2024-11-14T14:54:18,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41639 is added to blk_1073741835_1011 (size=5153) 2024-11-14T14:54:18,025 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/data/hbase/meta/1588230740/.tmp/ns/209fd144d478484bba2e155e1543c221 2024-11-14T14:54:18,043 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/data/hbase/meta/1588230740/.tmp/ns/209fd144d478484bba2e155e1543c221 as hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/data/hbase/meta/1588230740/ns/209fd144d478484bba2e155e1543c221 2024-11-14T14:54:18,058 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/data/hbase/meta/1588230740/ns/209fd144d478484bba2e155e1543c221, entries=2, sequenceid=6, filesize=5.0 K 2024-11-14T14:54:18,060 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=6, compaction requested=false 2024-11-14T14:54:18,060 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T14:54:18,067 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-14T14:54:18,068 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:54:18,068 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:54:18,068 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731596057980Running coprocessor pre-close hooks at 1731596057980Disabling compacts and flushes for region at 1731596057980Disabling writes for close at 1731596057980Obtaining lock to block concurrent updates at 1731596057981 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731596057981Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731596057981Flushing stores of hbase:meta,,1.1588230740 at 1731596057982 (+1 ms)Flushing 1588230740/ns: creating writer at 1731596057983 (+1 ms)Flushing 1588230740/ns: appending metadata at 1731596058009 (+26 ms)Flushing 1588230740/ns: closing flushed file at 1731596058009Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f4d51eb: reopening flushed file at 1731596058039 (+30 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=6, compaction requested=false at 1731596058060 (+21 ms)Writing region close event to WAL at 1731596058062 (+2 ms)Running coprocessor post-close hooks at 1731596058068 (+6 ms)Closed at 1731596058068 2024-11-14T14:54:18,068 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T14:54:18,180 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.HRegionServer(976): stopping server da1c6afcd1f9,40381,1731596056775; all regions closed. 2024-11-14T14:54:18,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:18,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:18,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:18,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:18,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:18,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34537 is added to blk_1073741834_1010 (size=1152) 2024-11-14T14:54:18,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41639 is added to blk_1073741834_1010 (size=1152) 2024-11-14T14:54:18,187 DEBUG [RS:0;da1c6afcd1f9:40381 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/oldWALs 2024-11-14T14:54:18,187 INFO [RS:0;da1c6afcd1f9:40381 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog da1c6afcd1f9%2C40381%2C1731596056775.meta:.meta(num 1731596057743) 2024-11-14T14:54:18,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:18,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:18,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:18,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:18,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:18,189 INFO [regionserver/da1c6afcd1f9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T14:54:18,189 INFO [regionserver/da1c6afcd1f9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T14:54:18,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41639 is added to blk_1073741833_1009 (size=93) 2024-11-14T14:54:18,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34537 is added to blk_1073741833_1009 (size=93) 2024-11-14T14:54:18,195 DEBUG [RS:0;da1c6afcd1f9:40381 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/oldWALs 2024-11-14T14:54:18,195 INFO [RS:0;da1c6afcd1f9:40381 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog da1c6afcd1f9%2C40381%2C1731596056775:(num 1731596057344) 2024-11-14T14:54:18,195 DEBUG [RS:0;da1c6afcd1f9:40381 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:54:18,195 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:54:18,195 INFO [RS:0;da1c6afcd1f9:40381 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:54:18,195 INFO [RS:0;da1c6afcd1f9:40381 {}] hbase.ChoreService(370): Chore service for: regionserver/da1c6afcd1f9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T14:54:18,196 INFO [RS:0;da1c6afcd1f9:40381 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:54:18,196 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:54:18,196 INFO [RS:0;da1c6afcd1f9:40381 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40381 2024-11-14T14:54:18,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/da1c6afcd1f9,40381,1731596056775 2024-11-14T14:54:18,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:54:18,198 INFO [RS:0;da1c6afcd1f9:40381 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:54:18,199 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [da1c6afcd1f9,40381,1731596056775] 2024-11-14T14:54:18,201 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/da1c6afcd1f9,40381,1731596056775 already deleted, retry=false 2024-11-14T14:54:18,201 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; da1c6afcd1f9,40381,1731596056775 expired; onlineServers=0 2024-11-14T14:54:18,201 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'da1c6afcd1f9,44379,1731596056719' ***** 2024-11-14T14:54:18,201 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T14:54:18,201 INFO [M:0;da1c6afcd1f9:44379 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:54:18,201 INFO [M:0;da1c6afcd1f9:44379 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:54:18,201 DEBUG [M:0;da1c6afcd1f9:44379 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T14:54:18,201 DEBUG [M:0;da1c6afcd1f9:44379 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T14:54:18,201 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T14:54:18,201 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731596057065 {}] cleaner.HFileCleaner(306): Exit Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731596057065,5,FailOnTimeoutGroup] 2024-11-14T14:54:18,202 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731596057063 {}] cleaner.HFileCleaner(306): Exit Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731596057063,5,FailOnTimeoutGroup] 2024-11-14T14:54:18,202 INFO [M:0;da1c6afcd1f9:44379 {}] hbase.ChoreService(370): Chore service for: master/da1c6afcd1f9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T14:54:18,202 INFO [M:0;da1c6afcd1f9:44379 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:54:18,202 DEBUG [M:0;da1c6afcd1f9:44379 {}] master.HMaster(1795): Stopping service threads 2024-11-14T14:54:18,202 INFO [M:0;da1c6afcd1f9:44379 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T14:54:18,202 INFO [M:0;da1c6afcd1f9:44379 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:54:18,202 INFO [M:0;da1c6afcd1f9:44379 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:54:18,202 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T14:54:18,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T14:54:18,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:18,203 DEBUG [M:0;da1c6afcd1f9:44379 {}] zookeeper.ZKUtil(347): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T14:54:18,203 WARN [M:0;da1c6afcd1f9:44379 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T14:54:18,204 INFO [M:0;da1c6afcd1f9:44379 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/.lastflushedseqids 2024-11-14T14:54:18,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41639 is added to blk_1073741836_1012 (size=99) 2024-11-14T14:54:18,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34537 is added to blk_1073741836_1012 (size=99) 2024-11-14T14:54:18,211 INFO [M:0;da1c6afcd1f9:44379 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:54:18,211 INFO [M:0;da1c6afcd1f9:44379 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T14:54:18,212 DEBUG [M:0;da1c6afcd1f9:44379 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:54:18,212 INFO [M:0;da1c6afcd1f9:44379 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:54:18,212 DEBUG [M:0;da1c6afcd1f9:44379 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:54:18,212 DEBUG [M:0;da1c6afcd1f9:44379 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:54:18,212 DEBUG [M:0;da1c6afcd1f9:44379 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:54:18,212 INFO [M:0;da1c6afcd1f9:44379 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-11-14T14:54:18,231 DEBUG [M:0;da1c6afcd1f9:44379 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/179f83bdb62545a8a674ef93923c1350 is 82, key is hbase:meta,,1/info:regioninfo/1731596057800/Put/seqid=0 2024-11-14T14:54:18,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41639 is added to blk_1073741837_1013 (size=5672) 2024-11-14T14:54:18,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34537 is added to blk_1073741837_1013 (size=5672) 2024-11-14T14:54:18,238 INFO [M:0;da1c6afcd1f9:44379 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/179f83bdb62545a8a674ef93923c1350 2024-11-14T14:54:18,261 DEBUG [M:0;da1c6afcd1f9:44379 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6c5eea063880477786eef88a3e12504b is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731596057832/Put/seqid=0 2024-11-14T14:54:18,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34537 is added to blk_1073741838_1014 (size=5275) 2024-11-14T14:54:18,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41639 is added to blk_1073741838_1014 (size=5275) 2024-11-14T14:54:18,269 INFO [M:0;da1c6afcd1f9:44379 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6c5eea063880477786eef88a3e12504b 2024-11-14T14:54:18,290 DEBUG [M:0;da1c6afcd1f9:44379 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5b3f7732c52a4bc88d8cb56f83add984 is 69, key is da1c6afcd1f9,40381,1731596056775/rs:state/1731596057159/Put/seqid=0 2024-11-14T14:54:18,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41639 is added to blk_1073741839_1015 (size=5156) 2024-11-14T14:54:18,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34537 is added to blk_1073741839_1015 (size=5156) 2024-11-14T14:54:18,297 INFO [M:0;da1c6afcd1f9:44379 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5b3f7732c52a4bc88d8cb56f83add984 2024-11-14T14:54:18,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:54:18,300 INFO [RS:0;da1c6afcd1f9:40381 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:54:18,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40381-0x101a7466de50001, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:54:18,300 INFO [RS:0;da1c6afcd1f9:40381 {}] regionserver.HRegionServer(1031): Exiting; stopping=da1c6afcd1f9,40381,1731596056775; zookeeper connection closed. 2024-11-14T14:54:18,300 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2975e0f5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2975e0f5 2024-11-14T14:54:18,300 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T14:54:18,324 DEBUG [M:0;da1c6afcd1f9:44379 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c839e226caac483087cf30c12e561835 is 52, key is load_balancer_on/state:d/1731596057967/Put/seqid=0 2024-11-14T14:54:18,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41639 is added to blk_1073741840_1016 (size=5056) 2024-11-14T14:54:18,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34537 is added to blk_1073741840_1016 (size=5056) 2024-11-14T14:54:18,330 INFO [M:0;da1c6afcd1f9:44379 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c839e226caac483087cf30c12e561835 2024-11-14T14:54:18,338 DEBUG [M:0;da1c6afcd1f9:44379 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/179f83bdb62545a8a674ef93923c1350 as hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/179f83bdb62545a8a674ef93923c1350 2024-11-14T14:54:18,345 INFO [M:0;da1c6afcd1f9:44379 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/179f83bdb62545a8a674ef93923c1350, entries=8, sequenceid=29, filesize=5.5 K 2024-11-14T14:54:18,346 DEBUG [M:0;da1c6afcd1f9:44379 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6c5eea063880477786eef88a3e12504b as hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6c5eea063880477786eef88a3e12504b 2024-11-14T14:54:18,353 INFO [M:0;da1c6afcd1f9:44379 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6c5eea063880477786eef88a3e12504b, entries=3, sequenceid=29, filesize=5.2 K 2024-11-14T14:54:18,354 DEBUG [M:0;da1c6afcd1f9:44379 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5b3f7732c52a4bc88d8cb56f83add984 as hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5b3f7732c52a4bc88d8cb56f83add984 2024-11-14T14:54:18,360 INFO [M:0;da1c6afcd1f9:44379 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5b3f7732c52a4bc88d8cb56f83add984, entries=1, sequenceid=29, filesize=5.0 K 2024-11-14T14:54:18,361 DEBUG [M:0;da1c6afcd1f9:44379 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c839e226caac483087cf30c12e561835 as hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c839e226caac483087cf30c12e561835 2024-11-14T14:54:18,367 INFO [M:0;da1c6afcd1f9:44379 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42069/user/jenkins/test-data/40b5bda1-c2cb-cb6b-403f-91ff207a1a6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c839e226caac483087cf30c12e561835, entries=1, sequenceid=29, filesize=4.9 K 2024-11-14T14:54:18,369 INFO [M:0;da1c6afcd1f9:44379 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 156ms, sequenceid=29, compaction requested=false 2024-11-14T14:54:18,370 INFO [M:0;da1c6afcd1f9:44379 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:54:18,370 DEBUG [M:0;da1c6afcd1f9:44379 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731596058211Disabling compacts and flushes for region at 1731596058211Disabling writes for close at 1731596058212 (+1 ms)Obtaining lock to block concurrent updates at 1731596058212Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731596058212Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731596058213 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731596058214 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731596058214Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731596058231 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731596058231Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731596058245 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731596058261 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731596058261Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731596058274 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731596058290 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731596058290Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731596058305 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731596058323 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731596058323Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36371ca3: reopening flushed file at 1731596058337 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7980a81b: reopening flushed file at 1731596058345 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b5df5c: reopening flushed file at 1731596058353 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bcf6783: reopening flushed file at 1731596058360 (+7 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 156ms, sequenceid=29, compaction requested=false at 1731596058369 (+9 ms)Writing region close event to WAL at 1731596058370 (+1 ms)Closed at 1731596058370 2024-11-14T14:54:18,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:18,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:18,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:18,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:18,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:18,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41639 is added to blk_1073741830_1006 (size=10311) 2024-11-14T14:54:18,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34537 is added to blk_1073741830_1006 (size=10311) 2024-11-14T14:54:18,375 INFO [M:0;da1c6afcd1f9:44379 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T14:54:18,375 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:54:18,375 INFO [M:0;da1c6afcd1f9:44379 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44379 2024-11-14T14:54:18,375 INFO [M:0;da1c6afcd1f9:44379 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:54:18,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:54:18,477 INFO [M:0;da1c6afcd1f9:44379 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:54:18,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44379-0x101a7466de50000, quorum=127.0.0.1:52554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:54:18,481 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7cebd4b6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:54:18,482 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1e6ff2f3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:54:18,482 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:54:18,482 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@18f27499{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:54:18,482 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d944f53{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/hadoop.log.dir/,STOPPED} 2024-11-14T14:54:18,484 WARN [BP-199397938-172.17.0.2-1731596055892 heartbeating to localhost/127.0.0.1:42069 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:54:18,484 WARN [BP-199397938-172.17.0.2-1731596055892 heartbeating to localhost/127.0.0.1:42069 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-199397938-172.17.0.2-1731596055892 (Datanode Uuid 9b52ebbe-807e-4c96-aabc-f6450c040781) service to localhost/127.0.0.1:42069 2024-11-14T14:54:18,485 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/cluster_6fc3c328-0a8c-0ecb-19f2-2869cdda52d1/data/data3/current/BP-199397938-172.17.0.2-1731596055892 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:54:18,485 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/cluster_6fc3c328-0a8c-0ecb-19f2-2869cdda52d1/data/data4/current/BP-199397938-172.17.0.2-1731596055892 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:54:18,485 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:54:18,485 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:54:18,485 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:54:18,491 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@61e52b83{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:54:18,491 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@29a18ee0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:54:18,491 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:54:18,491 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@670e4080{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:54:18,491 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@607b9bc6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/hadoop.log.dir/,STOPPED} 2024-11-14T14:54:18,493 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:54:18,493 WARN [BP-199397938-172.17.0.2-1731596055892 heartbeating to localhost/127.0.0.1:42069 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:54:18,493 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:54:18,493 WARN [BP-199397938-172.17.0.2-1731596055892 heartbeating to localhost/127.0.0.1:42069 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-199397938-172.17.0.2-1731596055892 (Datanode Uuid 5b21d4fd-8a38-4cb3-90e2-4bf2685a8269) service to localhost/127.0.0.1:42069 2024-11-14T14:54:18,494 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/cluster_6fc3c328-0a8c-0ecb-19f2-2869cdda52d1/data/data1/current/BP-199397938-172.17.0.2-1731596055892 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:54:18,495 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/cluster_6fc3c328-0a8c-0ecb-19f2-2869cdda52d1/data/data2/current/BP-199397938-172.17.0.2-1731596055892 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:54:18,495 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:54:18,502 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3235d5ba{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T14:54:18,503 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@347af0d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:54:18,503 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:54:18,503 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a69944b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:54:18,503 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a3c3ceb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/hadoop.log.dir/,STOPPED} 2024-11-14T14:54:18,510 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T14:54:18,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-14T14:54:18,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T14:54:18,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/hadoop.log.dir so I do NOT create it in target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea 2024-11-14T14:54:18,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6b021a8d-cc7f-cd25-dd0e-0475852eb57a/hadoop.tmp.dir so I do NOT create it in target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea 2024-11-14T14:54:18,532 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1, deleteOnExit=true 2024-11-14T14:54:18,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T14:54:18,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/test.cache.data in system properties and HBase conf 2024-11-14T14:54:18,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T14:54:18,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/hadoop.log.dir in system properties and HBase conf 2024-11-14T14:54:18,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T14:54:18,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T14:54:18,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T14:54:18,533 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T14:54:18,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T14:54:18,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T14:54:18,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T14:54:18,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T14:54:18,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T14:54:18,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T14:54:18,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T14:54:18,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T14:54:18,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T14:54:18,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/nfs.dump.dir in system properties and HBase conf 2024-11-14T14:54:18,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/java.io.tmpdir in system properties and HBase conf 2024-11-14T14:54:18,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T14:54:18,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T14:54:18,535 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T14:54:18,554 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-14T14:54:18,625 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:54:18,632 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:54:18,637 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:54:18,637 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:54:18,637 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:54:18,638 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:54:18,639 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2305029e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:54:18,639 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6a488aac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:54:18,765 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@195100a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/java.io.tmpdir/jetty-localhost-43809-hadoop-hdfs-3_4_1-tests_jar-_-any-15070906502551094380/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T14:54:18,766 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1e349773{HTTP/1.1, (http/1.1)}{localhost:43809} 2024-11-14T14:54:18,766 INFO [Time-limited test {}] server.Server(415): Started @104388ms 2024-11-14T14:54:18,780 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-14T14:54:18,850 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:54:18,853 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:54:18,856 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:54:18,856 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:54:18,856 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T14:54:18,857 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48743db4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:54:18,857 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@257cf4bb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:54:18,974 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@692b8c40{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/java.io.tmpdir/jetty-localhost-33175-hadoop-hdfs-3_4_1-tests_jar-_-any-1621725518526560993/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:54:18,975 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1369ef4b{HTTP/1.1, (http/1.1)}{localhost:33175} 2024-11-14T14:54:18,975 INFO [Time-limited test {}] server.Server(415): Started @104597ms 2024-11-14T14:54:18,977 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:54:19,018 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:54:19,021 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:54:19,022 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:54:19,022 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:54:19,022 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T14:54:19,023 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1ec1a06e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:54:19,023 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69bbaec1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:54:19,076 WARN [Thread-658 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data2/current/BP-1040518630-172.17.0.2-1731596058577/current, will proceed with Du for space computation calculation, 2024-11-14T14:54:19,076 WARN [Thread-657 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data1/current/BP-1040518630-172.17.0.2-1731596058577/current, will proceed with Du for space computation calculation, 2024-11-14T14:54:19,102 WARN [Thread-636 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:54:19,105 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x67cae61ddee519bb with lease ID 0x2ad170a756447202: Processing first storage report for DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea from datanode DatanodeRegistration(127.0.0.1:33793, datanodeUuid=0ba76aad-cfa9-4a36-afa7-cde9f51172e1, infoPort=40049, infoSecurePort=0, ipcPort=43867, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577) 2024-11-14T14:54:19,105 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x67cae61ddee519bb with lease ID 0x2ad170a756447202: from storage DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea node DatanodeRegistration(127.0.0.1:33793, datanodeUuid=0ba76aad-cfa9-4a36-afa7-cde9f51172e1, infoPort=40049, infoSecurePort=0, ipcPort=43867, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:54:19,105 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x67cae61ddee519bb with lease ID 0x2ad170a756447202: Processing first storage report for DS-f2e7aa96-7c6b-42e2-9b7a-ac347fd45329 from datanode DatanodeRegistration(127.0.0.1:33793, datanodeUuid=0ba76aad-cfa9-4a36-afa7-cde9f51172e1, infoPort=40049, infoSecurePort=0, ipcPort=43867, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577) 2024-11-14T14:54:19,106 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x67cae61ddee519bb with lease ID 0x2ad170a756447202: from storage DS-f2e7aa96-7c6b-42e2-9b7a-ac347fd45329 node DatanodeRegistration(127.0.0.1:33793, datanodeUuid=0ba76aad-cfa9-4a36-afa7-cde9f51172e1, infoPort=40049, infoSecurePort=0, ipcPort=43867, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:54:19,150 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@f7d5275{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/java.io.tmpdir/jetty-localhost-40435-hadoop-hdfs-3_4_1-tests_jar-_-any-15116153522156730697/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:54:19,151 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5f783084{HTTP/1.1, (http/1.1)}{localhost:40435} 2024-11-14T14:54:19,151 INFO [Time-limited test {}] server.Server(415): Started @104772ms 2024-11-14T14:54:19,152 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:54:19,193 INFO [regionserver/da1c6afcd1f9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:54:19,258 WARN [Thread-684 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data4/current/BP-1040518630-172.17.0.2-1731596058577/current, will proceed with Du for space computation calculation, 2024-11-14T14:54:19,258 WARN [Thread-683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data3/current/BP-1040518630-172.17.0.2-1731596058577/current, will proceed with Du for space computation calculation, 2024-11-14T14:54:19,289 WARN [Thread-672 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:54:19,292 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcc23761eb56b7ac3 with lease ID 0x2ad170a756447203: Processing first storage report for DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015 from datanode DatanodeRegistration(127.0.0.1:35719, datanodeUuid=ed0576a8-2ece-4054-b155-ea617541c969, infoPort=38019, infoSecurePort=0, ipcPort=45771, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577) 2024-11-14T14:54:19,292 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcc23761eb56b7ac3 with lease ID 0x2ad170a756447203: from storage DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015 node DatanodeRegistration(127.0.0.1:35719, datanodeUuid=ed0576a8-2ece-4054-b155-ea617541c969, infoPort=38019, infoSecurePort=0, ipcPort=45771, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:54:19,292 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xcc23761eb56b7ac3 with lease ID 0x2ad170a756447203: Processing first storage report for DS-0771e6af-5153-420f-a8b9-069a2555649a from datanode DatanodeRegistration(127.0.0.1:35719, datanodeUuid=ed0576a8-2ece-4054-b155-ea617541c969, infoPort=38019, infoSecurePort=0, ipcPort=45771, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577) 2024-11-14T14:54:19,292 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xcc23761eb56b7ac3 with lease ID 0x2ad170a756447203: from storage DS-0771e6af-5153-420f-a8b9-069a2555649a node DatanodeRegistration(127.0.0.1:35719, datanodeUuid=ed0576a8-2ece-4054-b155-ea617541c969, infoPort=38019, infoSecurePort=0, ipcPort=45771, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:54:19,384 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea 2024-11-14T14:54:19,387 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/zookeeper_0, clientPort=60317, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T14:54:19,388 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60317 2024-11-14T14:54:19,389 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:54:19,390 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:54:19,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33793 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:54:19,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35719 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:54:19,403 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7 with version=8 2024-11-14T14:54:19,403 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/hbase-staging 2024-11-14T14:54:19,405 INFO [Time-limited test {}] client.ConnectionUtils(128): master/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:54:19,405 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:54:19,405 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:54:19,406 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:54:19,406 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:54:19,406 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:54:19,406 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T14:54:19,406 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:54:19,407 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40173 2024-11-14T14:54:19,408 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40173 connecting to ZooKeeper ensemble=127.0.0.1:60317 2024-11-14T14:54:19,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:401730x0, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:54:19,414 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40173-0x101a74678630000 connected 2024-11-14T14:54:19,440 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:54:19,442 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:54:19,446 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:54:19,446 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7, hbase.cluster.distributed=false 2024-11-14T14:54:19,450 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:54:19,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40173 2024-11-14T14:54:19,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40173 2024-11-14T14:54:19,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40173 2024-11-14T14:54:19,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40173 2024-11-14T14:54:19,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40173 2024-11-14T14:54:19,482 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:54:19,483 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:54:19,483 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:54:19,483 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:54:19,483 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:54:19,483 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:54:19,483 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T14:54:19,483 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:54:19,484 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40527 2024-11-14T14:54:19,486 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40527 connecting to ZooKeeper ensemble=127.0.0.1:60317 2024-11-14T14:54:19,487 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:54:19,490 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:54:19,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:405270x0, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:54:19,497 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:405270x0, quorum=127.0.0.1:60317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:54:19,497 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T14:54:19,500 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40527-0x101a74678630001 connected 2024-11-14T14:54:19,500 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T14:54:19,501 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T14:54:19,502 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:54:19,503 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40527 2024-11-14T14:54:19,503 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40527 2024-11-14T14:54:19,503 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40527 2024-11-14T14:54:19,506 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40527 2024-11-14T14:54:19,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40527 2024-11-14T14:54:19,521 DEBUG [M:0;da1c6afcd1f9:40173 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;da1c6afcd1f9:40173 2024-11-14T14:54:19,522 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/da1c6afcd1f9,40173,1731596059405 2024-11-14T14:54:19,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:54:19,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:54:19,525 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/da1c6afcd1f9,40173,1731596059405 2024-11-14T14:54:19,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T14:54:19,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:19,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:19,527 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T14:54:19,528 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/da1c6afcd1f9,40173,1731596059405 from backup master directory 2024-11-14T14:54:19,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/da1c6afcd1f9,40173,1731596059405 2024-11-14T14:54:19,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:54:19,529 WARN [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:54:19,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:54:19,530 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=da1c6afcd1f9,40173,1731596059405 2024-11-14T14:54:19,537 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/hbase.id] with ID: c6a0dae0-03e5-4bba-8708-3e09551f71f9 2024-11-14T14:54:19,537 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/.tmp/hbase.id 2024-11-14T14:54:19,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35719 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:54:19,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33793 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:54:19,551 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/.tmp/hbase.id]:[hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/hbase.id] 2024-11-14T14:54:19,567 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:54:19,567 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T14:54:19,569 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-14T14:54:19,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:19,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:19,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35719 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:54:19,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33793 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:54:19,587 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T14:54:19,588 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T14:54:19,589 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:54:19,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33793 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:54:19,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35719 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:54:19,608 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store 2024-11-14T14:54:19,617 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:54:19,617 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T14:54:19,618 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T14:54:19,619 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-11-14T14:54:19,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33793 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:54:19,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35719 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:54:19,627 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:54:19,628 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:54:19,628 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:54:19,628 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:54:19,628 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:54:19,628 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:54:19,628 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:54:19,628 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731596059628Disabling compacts and flushes for region at 1731596059628Disabling writes for close at 1731596059628Writing region close event to WAL at 1731596059628Closed at 1731596059628 2024-11-14T14:54:19,629 WARN [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/.initializing 2024-11-14T14:54:19,629 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/WALs/da1c6afcd1f9,40173,1731596059405 2024-11-14T14:54:19,635 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C40173%2C1731596059405, suffix=, logDir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/WALs/da1c6afcd1f9,40173,1731596059405, archiveDir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/oldWALs, maxLogs=10 2024-11-14T14:54:19,635 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C40173%2C1731596059405.1731596059635 2024-11-14T14:54:19,643 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/WALs/da1c6afcd1f9,40173,1731596059405/da1c6afcd1f9%2C40173%2C1731596059405.1731596059635 2024-11-14T14:54:19,648 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:38019:38019)] 2024-11-14T14:54:19,652 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:54:19,653 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:54:19,653 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:19,653 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:19,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:19,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T14:54:19,657 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:19,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:54:19,658 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:19,659 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T14:54:19,659 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:19,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:54:19,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:19,661 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T14:54:19,661 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:19,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:54:19,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:19,663 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T14:54:19,663 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:19,664 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:54:19,664 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:19,665 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:19,665 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:19,667 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:19,667 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:19,668 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T14:54:19,670 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:54:19,672 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:54:19,673 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=698991, jitterRate=-0.11118744313716888}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T14:54:19,674 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731596059653Initializing all the Stores at 1731596059654 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596059654Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596059655 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596059655Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596059655Cleaning up temporary data from old regions at 1731596059667 (+12 ms)Region opened successfully at 1731596059674 (+7 ms) 2024-11-14T14:54:19,675 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T14:54:19,679 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ebf54a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:54:19,680 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T14:54:19,680 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T14:54:19,680 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T14:54:19,680 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:54:19,681 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:54:19,681 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-14T14:54:19,681 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T14:54:19,683 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T14:54:19,684 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T14:54:19,686 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T14:54:19,686 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T14:54:19,687 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T14:54:19,688 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T14:54:19,688 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T14:54:19,689 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T14:54:19,691 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T14:54:19,692 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T14:54:19,693 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T14:54:19,696 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T14:54:19,697 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T14:54:19,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:54:19,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:54:19,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:19,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:19,700 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=da1c6afcd1f9,40173,1731596059405, sessionid=0x101a74678630000, setting cluster-up flag (Was=false) 2024-11-14T14:54:19,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:19,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:19,709 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T14:54:19,711 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da1c6afcd1f9,40173,1731596059405 2024-11-14T14:54:19,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:19,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:19,720 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T14:54:19,721 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da1c6afcd1f9,40173,1731596059405 2024-11-14T14:54:19,723 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T14:54:19,725 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:54:19,725 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T14:54:19,725 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T14:54:19,726 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: da1c6afcd1f9,40173,1731596059405 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T14:54:19,727 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:54:19,727 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:54:19,727 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:54:19,727 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:54:19,727 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/da1c6afcd1f9:0, corePoolSize=10, maxPoolSize=10 2024-11-14T14:54:19,727 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:19,727 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:54:19,727 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:19,729 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:54:19,729 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T14:54:19,731 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:19,731 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T14:54:19,732 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731596089732 2024-11-14T14:54:19,762 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T14:54:19,762 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T14:54:19,762 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T14:54:19,762 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T14:54:19,762 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T14:54:19,762 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T14:54:19,763 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:19,763 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T14:54:19,763 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T14:54:19,763 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T14:54:19,764 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T14:54:19,764 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T14:54:19,772 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731596059764,5,FailOnTimeoutGroup] 2024-11-14T14:54:19,775 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731596059772,5,FailOnTimeoutGroup] 2024-11-14T14:54:19,775 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:19,776 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T14:54:19,776 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:19,776 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:19,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33793 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:54:19,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35719 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:54:19,779 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T14:54:19,780 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7 2024-11-14T14:54:19,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33793 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:54:19,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35719 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:54:19,794 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:54:19,795 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:54:19,797 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:54:19,797 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:19,797 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:54:19,798 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:54:19,799 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:54:19,799 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:19,799 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:54:19,799 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:54:19,801 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:54:19,801 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:19,801 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:54:19,801 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:54:19,803 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:54:19,803 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:19,803 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:54:19,803 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:54:19,804 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740 2024-11-14T14:54:19,805 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740 2024-11-14T14:54:19,806 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:54:19,806 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:54:19,807 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T14:54:19,808 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:54:19,811 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.HRegionServer(746): ClusterId : c6a0dae0-03e5-4bba-8708-3e09551f71f9 2024-11-14T14:54:19,811 DEBUG [RS:0;da1c6afcd1f9:40527 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T14:54:19,814 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:54:19,815 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:54:19,816 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=737434, jitterRate=-0.062304720282554626}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T14:54:19,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731596059794Initializing all the Stores at 1731596059795 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596059795Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596059795Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596059795Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596059795Cleaning up temporary data from old regions at 1731596059806 (+11 ms)Region opened successfully at 1731596059817 (+11 ms) 2024-11-14T14:54:19,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:54:19,817 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:54:19,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:54:19,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:54:19,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:54:19,818 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:54:19,818 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731596059817Disabling compacts and flushes for region at 1731596059817Disabling writes for close at 1731596059817Writing region close event to WAL at 1731596059818 (+1 ms)Closed at 1731596059818 2024-11-14T14:54:19,820 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:54:19,819 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:54:19,820 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T14:54:19,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T14:54:19,821 DEBUG [RS:0;da1c6afcd1f9:40527 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T14:54:19,822 DEBUG [RS:0;da1c6afcd1f9:40527 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T14:54:19,822 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:54:19,824 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T14:54:19,827 DEBUG [RS:0;da1c6afcd1f9:40527 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T14:54:19,828 DEBUG [RS:0;da1c6afcd1f9:40527 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19389fbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:54:19,845 DEBUG [RS:0;da1c6afcd1f9:40527 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;da1c6afcd1f9:40527 2024-11-14T14:54:19,846 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T14:54:19,846 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T14:54:19,846 DEBUG [RS:0;da1c6afcd1f9:40527 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T14:54:19,847 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.HRegionServer(2659): reportForDuty to master=da1c6afcd1f9,40173,1731596059405 with port=40527, startcode=1731596059482 2024-11-14T14:54:19,847 DEBUG [RS:0;da1c6afcd1f9:40527 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T14:54:19,850 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33229, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T14:54:19,850 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40173 {}] master.ServerManager(363): Checking decommissioned status of RegionServer da1c6afcd1f9,40527,1731596059482 2024-11-14T14:54:19,850 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40173 {}] master.ServerManager(517): Registering regionserver=da1c6afcd1f9,40527,1731596059482 2024-11-14T14:54:19,852 DEBUG [RS:0;da1c6afcd1f9:40527 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7 2024-11-14T14:54:19,852 DEBUG [RS:0;da1c6afcd1f9:40527 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35595 2024-11-14T14:54:19,852 DEBUG [RS:0;da1c6afcd1f9:40527 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T14:54:19,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:54:19,855 DEBUG [RS:0;da1c6afcd1f9:40527 {}] zookeeper.ZKUtil(111): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/da1c6afcd1f9,40527,1731596059482 2024-11-14T14:54:19,855 WARN [RS:0;da1c6afcd1f9:40527 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:54:19,855 INFO [RS:0;da1c6afcd1f9:40527 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:54:19,856 DEBUG [RS:0;da1c6afcd1f9:40527 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482 2024-11-14T14:54:19,857 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [da1c6afcd1f9,40527,1731596059482] 2024-11-14T14:54:19,862 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T14:54:19,864 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T14:54:19,864 INFO [RS:0;da1c6afcd1f9:40527 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T14:54:19,864 INFO [RS:0;da1c6afcd1f9:40527 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:19,868 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T14:54:19,869 INFO [RS:0;da1c6afcd1f9:40527 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T14:54:19,869 INFO [RS:0;da1c6afcd1f9:40527 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:19,869 DEBUG [RS:0;da1c6afcd1f9:40527 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:19,870 DEBUG [RS:0;da1c6afcd1f9:40527 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:19,870 DEBUG [RS:0;da1c6afcd1f9:40527 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:19,870 DEBUG [RS:0;da1c6afcd1f9:40527 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:19,870 DEBUG [RS:0;da1c6afcd1f9:40527 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:19,870 DEBUG [RS:0;da1c6afcd1f9:40527 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:54:19,870 DEBUG [RS:0;da1c6afcd1f9:40527 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:19,870 DEBUG [RS:0;da1c6afcd1f9:40527 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:19,870 DEBUG [RS:0;da1c6afcd1f9:40527 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:19,870 DEBUG [RS:0;da1c6afcd1f9:40527 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:19,870 DEBUG [RS:0;da1c6afcd1f9:40527 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:19,871 DEBUG [RS:0;da1c6afcd1f9:40527 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:19,871 DEBUG [RS:0;da1c6afcd1f9:40527 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:54:19,871 DEBUG [RS:0;da1c6afcd1f9:40527 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:54:19,872 INFO [RS:0;da1c6afcd1f9:40527 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:19,873 INFO [RS:0;da1c6afcd1f9:40527 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:19,873 INFO [RS:0;da1c6afcd1f9:40527 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:19,873 INFO [RS:0;da1c6afcd1f9:40527 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:19,873 INFO [RS:0;da1c6afcd1f9:40527 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:19,873 INFO [RS:0;da1c6afcd1f9:40527 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40527,1731596059482-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:54:19,893 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T14:54:19,894 INFO [RS:0;da1c6afcd1f9:40527 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40527,1731596059482-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:19,894 INFO [RS:0;da1c6afcd1f9:40527 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:19,894 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.Replication(171): da1c6afcd1f9,40527,1731596059482 started 2024-11-14T14:54:19,909 INFO [RS:0;da1c6afcd1f9:40527 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:19,909 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.HRegionServer(1482): Serving as da1c6afcd1f9,40527,1731596059482, RpcServer on da1c6afcd1f9/172.17.0.2:40527, sessionid=0x101a74678630001 2024-11-14T14:54:19,909 DEBUG [RS:0;da1c6afcd1f9:40527 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T14:54:19,909 DEBUG [RS:0;da1c6afcd1f9:40527 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager da1c6afcd1f9,40527,1731596059482 2024-11-14T14:54:19,909 DEBUG [RS:0;da1c6afcd1f9:40527 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,40527,1731596059482' 2024-11-14T14:54:19,909 DEBUG [RS:0;da1c6afcd1f9:40527 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T14:54:19,910 DEBUG [RS:0;da1c6afcd1f9:40527 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T14:54:19,910 DEBUG [RS:0;da1c6afcd1f9:40527 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T14:54:19,910 DEBUG [RS:0;da1c6afcd1f9:40527 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T14:54:19,910 DEBUG [RS:0;da1c6afcd1f9:40527 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager da1c6afcd1f9,40527,1731596059482 2024-11-14T14:54:19,910 DEBUG [RS:0;da1c6afcd1f9:40527 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,40527,1731596059482' 2024-11-14T14:54:19,911 DEBUG [RS:0;da1c6afcd1f9:40527 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T14:54:19,911 DEBUG [RS:0;da1c6afcd1f9:40527 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T14:54:19,911 DEBUG [RS:0;da1c6afcd1f9:40527 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T14:54:19,912 INFO [RS:0;da1c6afcd1f9:40527 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T14:54:19,912 INFO [RS:0;da1c6afcd1f9:40527 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T14:54:19,974 WARN [da1c6afcd1f9:40173 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-14T14:54:20,014 INFO [RS:0;da1c6afcd1f9:40527 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C40527%2C1731596059482, suffix=, logDir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482, archiveDir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/oldWALs, maxLogs=32 2024-11-14T14:54:20,015 INFO [RS:0;da1c6afcd1f9:40527 {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C40527%2C1731596059482.1731596060015 2024-11-14T14:54:20,030 INFO [RS:0;da1c6afcd1f9:40527 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596060015 2024-11-14T14:54:20,044 DEBUG [RS:0;da1c6afcd1f9:40527 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38019:38019),(127.0.0.1/127.0.0.1:40049:40049)] 2024-11-14T14:54:20,225 DEBUG [da1c6afcd1f9:40173 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T14:54:20,226 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=da1c6afcd1f9,40527,1731596059482 2024-11-14T14:54:20,227 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40527,1731596059482, state=OPENING 2024-11-14T14:54:20,229 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T14:54:20,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:20,231 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:20,232 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:54:20,232 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:54:20,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40527,1731596059482}] 2024-11-14T14:54:20,232 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:54:20,333 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-14T14:54:20,335 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:54:20,352 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:54:20,354 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:54:20,354 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:54:20,386 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T14:54:20,388 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40243, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T14:54:20,393 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T14:54:20,393 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:54:20,395 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C40527%2C1731596059482.meta, suffix=.meta, logDir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482, archiveDir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/oldWALs, maxLogs=32 2024-11-14T14:54:20,396 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta 2024-11-14T14:54:20,402 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta 2024-11-14T14:54:20,403 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38019:38019),(127.0.0.1/127.0.0.1:40049:40049)] 2024-11-14T14:54:20,404 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:54:20,404 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T14:54:20,404 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T14:54:20,404 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T14:54:20,404 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T14:54:20,404 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:54:20,405 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T14:54:20,405 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T14:54:20,406 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:54:20,407 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:54:20,407 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:20,408 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:54:20,408 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:54:20,409 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:54:20,409 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:20,410 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:54:20,410 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:54:20,411 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:54:20,411 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:20,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:54:20,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:54:20,413 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:54:20,413 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:20,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:54:20,413 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:54:20,414 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740 2024-11-14T14:54:20,416 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740 2024-11-14T14:54:20,417 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:54:20,418 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:54:20,418 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T14:54:20,420 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:54:20,421 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=881680, jitterRate=0.12111522257328033}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T14:54:20,421 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T14:54:20,422 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731596060405Writing region info on filesystem at 1731596060405Initializing all the Stores at 1731596060406 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596060406Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596060406Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596060406Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596060406Cleaning up temporary data from old regions at 1731596060418 (+12 ms)Running coprocessor post-open hooks at 1731596060421 (+3 ms)Region opened successfully at 1731596060422 (+1 ms) 2024-11-14T14:54:20,424 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731596060386 2024-11-14T14:54:20,427 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T14:54:20,427 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T14:54:20,428 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,40527,1731596059482 2024-11-14T14:54:20,429 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,40527,1731596059482, state=OPEN 2024-11-14T14:54:20,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:54:20,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:54:20,434 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:54:20,434 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:54:20,434 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40527,1731596059482 2024-11-14T14:54:20,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T14:54:20,438 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,40527,1731596059482 in 202 msec 2024-11-14T14:54:20,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T14:54:20,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 618 msec 2024-11-14T14:54:20,442 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:54:20,442 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T14:54:20,444 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:54:20,444 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,40527,1731596059482, seqNum=-1] 2024-11-14T14:54:20,444 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:54:20,446 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52047, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:54:20,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 728 msec 2024-11-14T14:54:20,453 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731596060453, completionTime=-1 2024-11-14T14:54:20,453 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T14:54:20,454 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:54:20,455 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-14T14:54:20,455 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731596120455 2024-11-14T14:54:20,455 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731596180455 2024-11-14T14:54:20,456 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-14T14:54:20,456 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40173,1731596059405-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:20,456 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40173,1731596059405-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:20,456 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40173,1731596059405-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:20,456 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-da1c6afcd1f9:40173, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:20,456 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:20,457 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:20,458 DEBUG [master/da1c6afcd1f9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T14:54:20,462 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.932sec 2024-11-14T14:54:20,462 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T14:54:20,462 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T14:54:20,462 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T14:54:20,462 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T14:54:20,462 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T14:54:20,462 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40173,1731596059405-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:54:20,462 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40173,1731596059405-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T14:54:20,465 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T14:54:20,465 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T14:54:20,465 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40173,1731596059405-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:20,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@597807df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:54:20,511 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request da1c6afcd1f9,40173,-1 for getting cluster id 2024-11-14T14:54:20,511 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T14:54:20,513 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6a0dae0-03e5-4bba-8708-3e09551f71f9' 2024-11-14T14:54:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T14:54:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6a0dae0-03e5-4bba-8708-3e09551f71f9" 2024-11-14T14:54:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a72ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:54:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [da1c6afcd1f9,40173,-1] 2024-11-14T14:54:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T14:54:20,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:54:20,517 INFO [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T14:54:20,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3efce601, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:54:20,518 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:54:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,40527,1731596059482, seqNum=-1] 2024-11-14T14:54:20,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:54:20,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:54:20,523 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=da1c6afcd1f9,40173,1731596059405 2024-11-14T14:54:20,524 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:54:20,527 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-14T14:54:20,544 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:54:20,545 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:54:20,545 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:54:20,545 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:54:20,545 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:54:20,545 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:54:20,545 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T14:54:20,545 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:54:20,546 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34075 2024-11-14T14:54:20,548 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34075 connecting to ZooKeeper ensemble=127.0.0.1:60317 2024-11-14T14:54:20,549 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:54:20,550 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:54:20,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:340750x0, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:54:20,560 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:340750x0, quorum=127.0.0.1:60317, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-14T14:54:20,560 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-14T14:54:20,560 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34075-0x101a74678630002 connected 2024-11-14T14:54:20,560 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T14:54:20,568 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T14:54:20,569 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:34075-0x101a74678630002, quorum=127.0.0.1:60317, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T14:54:20,571 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34075-0x101a74678630002, quorum=127.0.0.1:60317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:54:20,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34075 2024-11-14T14:54:20,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34075 2024-11-14T14:54:20,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34075 2024-11-14T14:54:20,578 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34075 2024-11-14T14:54:20,578 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34075 2024-11-14T14:54:20,581 INFO [RS:1;da1c6afcd1f9:34075 {}] regionserver.HRegionServer(746): ClusterId : c6a0dae0-03e5-4bba-8708-3e09551f71f9 2024-11-14T14:54:20,581 DEBUG [RS:1;da1c6afcd1f9:34075 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T14:54:20,584 DEBUG [RS:1;da1c6afcd1f9:34075 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T14:54:20,584 DEBUG [RS:1;da1c6afcd1f9:34075 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T14:54:20,590 DEBUG [RS:1;da1c6afcd1f9:34075 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T14:54:20,591 DEBUG [RS:1;da1c6afcd1f9:34075 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c56f5f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:54:20,607 DEBUG [RS:1;da1c6afcd1f9:34075 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;da1c6afcd1f9:34075 2024-11-14T14:54:20,607 INFO [RS:1;da1c6afcd1f9:34075 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T14:54:20,607 INFO [RS:1;da1c6afcd1f9:34075 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T14:54:20,607 DEBUG [RS:1;da1c6afcd1f9:34075 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T14:54:20,608 INFO [RS:1;da1c6afcd1f9:34075 {}] regionserver.HRegionServer(2659): reportForDuty to master=da1c6afcd1f9,40173,1731596059405 with port=34075, startcode=1731596060544 2024-11-14T14:54:20,608 DEBUG [RS:1;da1c6afcd1f9:34075 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T14:54:20,616 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52675, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T14:54:20,616 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40173 {}] master.ServerManager(363): Checking decommissioned status of RegionServer da1c6afcd1f9,34075,1731596060544 2024-11-14T14:54:20,616 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40173 {}] master.ServerManager(517): Registering regionserver=da1c6afcd1f9,34075,1731596060544 2024-11-14T14:54:20,619 DEBUG [RS:1;da1c6afcd1f9:34075 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7 2024-11-14T14:54:20,619 DEBUG [RS:1;da1c6afcd1f9:34075 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35595 2024-11-14T14:54:20,619 DEBUG [RS:1;da1c6afcd1f9:34075 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T14:54:20,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:54:20,624 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [da1c6afcd1f9,34075,1731596060544] 2024-11-14T14:54:20,625 DEBUG [RS:1;da1c6afcd1f9:34075 {}] zookeeper.ZKUtil(111): regionserver:34075-0x101a74678630002, quorum=127.0.0.1:60317, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/da1c6afcd1f9,34075,1731596060544 2024-11-14T14:54:20,625 WARN [RS:1;da1c6afcd1f9:34075 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:54:20,625 INFO [RS:1;da1c6afcd1f9:34075 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:54:20,625 DEBUG [RS:1;da1c6afcd1f9:34075 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544 2024-11-14T14:54:20,633 INFO [RS:1;da1c6afcd1f9:34075 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T14:54:20,640 INFO [RS:1;da1c6afcd1f9:34075 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T14:54:20,642 INFO [RS:1;da1c6afcd1f9:34075 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T14:54:20,642 INFO [RS:1;da1c6afcd1f9:34075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:20,642 INFO [RS:1;da1c6afcd1f9:34075 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T14:54:20,644 INFO [RS:1;da1c6afcd1f9:34075 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T14:54:20,644 INFO [RS:1;da1c6afcd1f9:34075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:20,644 DEBUG [RS:1;da1c6afcd1f9:34075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:20,644 DEBUG [RS:1;da1c6afcd1f9:34075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:20,644 DEBUG [RS:1;da1c6afcd1f9:34075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:20,644 DEBUG [RS:1;da1c6afcd1f9:34075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:20,644 DEBUG [RS:1;da1c6afcd1f9:34075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:20,644 DEBUG [RS:1;da1c6afcd1f9:34075 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:54:20,645 DEBUG [RS:1;da1c6afcd1f9:34075 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:20,645 DEBUG [RS:1;da1c6afcd1f9:34075 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:20,645 DEBUG [RS:1;da1c6afcd1f9:34075 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:20,645 DEBUG [RS:1;da1c6afcd1f9:34075 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:20,645 DEBUG [RS:1;da1c6afcd1f9:34075 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:20,645 DEBUG [RS:1;da1c6afcd1f9:34075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:54:20,645 DEBUG [RS:1;da1c6afcd1f9:34075 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:54:20,645 DEBUG [RS:1;da1c6afcd1f9:34075 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:54:20,662 INFO [RS:1;da1c6afcd1f9:34075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:20,662 INFO [RS:1;da1c6afcd1f9:34075 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:20,662 INFO [RS:1;da1c6afcd1f9:34075 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:20,662 INFO [RS:1;da1c6afcd1f9:34075 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:20,663 INFO [RS:1;da1c6afcd1f9:34075 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:20,663 INFO [RS:1;da1c6afcd1f9:34075 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,34075,1731596060544-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:54:20,684 INFO [RS:1;da1c6afcd1f9:34075 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T14:54:20,684 INFO [RS:1;da1c6afcd1f9:34075 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,34075,1731596060544-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:20,685 INFO [RS:1;da1c6afcd1f9:34075 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:20,685 INFO [RS:1;da1c6afcd1f9:34075 {}] regionserver.Replication(171): da1c6afcd1f9,34075,1731596060544 started 2024-11-14T14:54:20,704 INFO [RS:1;da1c6afcd1f9:34075 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:54:20,704 INFO [RS:1;da1c6afcd1f9:34075 {}] regionserver.HRegionServer(1482): Serving as da1c6afcd1f9,34075,1731596060544, RpcServer on da1c6afcd1f9/172.17.0.2:34075, sessionid=0x101a74678630002 2024-11-14T14:54:20,705 DEBUG [RS:1;da1c6afcd1f9:34075 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T14:54:20,705 DEBUG [RS:1;da1c6afcd1f9:34075 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager da1c6afcd1f9,34075,1731596060544 2024-11-14T14:54:20,705 DEBUG [RS:1;da1c6afcd1f9:34075 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,34075,1731596060544' 2024-11-14T14:54:20,705 DEBUG [RS:1;da1c6afcd1f9:34075 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T14:54:20,705 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;da1c6afcd1f9:34075,5,FailOnTimeoutGroup] 2024-11-14T14:54:20,705 INFO [Time-limited test {}] wal.TestLogRolling(207): Replication=2 2024-11-14T14:54:20,705 DEBUG [RS:1;da1c6afcd1f9:34075 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T14:54:20,706 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T14:54:20,706 DEBUG [RS:1;da1c6afcd1f9:34075 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T14:54:20,706 DEBUG [RS:1;da1c6afcd1f9:34075 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T14:54:20,706 DEBUG [RS:1;da1c6afcd1f9:34075 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager da1c6afcd1f9,34075,1731596060544 2024-11-14T14:54:20,706 DEBUG [RS:1;da1c6afcd1f9:34075 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,34075,1731596060544' 2024-11-14T14:54:20,706 DEBUG [RS:1;da1c6afcd1f9:34075 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T14:54:20,707 DEBUG [RS:1;da1c6afcd1f9:34075 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T14:54:20,707 DEBUG [RS:1;da1c6afcd1f9:34075 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T14:54:20,707 INFO [RS:1;da1c6afcd1f9:34075 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T14:54:20,707 INFO [RS:1;da1c6afcd1f9:34075 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T14:54:20,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is da1c6afcd1f9,40173,1731596059405 2024-11-14T14:54:20,707 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@f09b82f 2024-11-14T14:54:20,708 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T14:54:20,711 INFO [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T14:54:20,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-14T14:54:20,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-14T14:54:20,712 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T14:54:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath 2024-11-14T14:54:20,716 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T14:54:20,716 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:20,717 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnDatanodeDeath" procId is: 4 2024-11-14T14:54:20,718 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T14:54:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:54:20,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35719 is added to blk_1073741835_1011 (size=393) 2024-11-14T14:54:20,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33793 is added to blk_1073741835_1011 (size=393) 2024-11-14T14:54:20,736 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c2979e3a8fec3327dfa64983b4f61c28, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7 2024-11-14T14:54:20,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35719 is added to blk_1073741836_1012 (size=76) 2024-11-14T14:54:20,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33793 is added to blk_1073741836_1012 (size=76) 2024-11-14T14:54:20,764 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:54:20,764 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing c2979e3a8fec3327dfa64983b4f61c28, disabling compactions & flushes 2024-11-14T14:54:20,764 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28. 2024-11-14T14:54:20,764 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28. 2024-11-14T14:54:20,764 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28. after waiting 0 ms 2024-11-14T14:54:20,764 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28. 2024-11-14T14:54:20,764 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28. 2024-11-14T14:54:20,764 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for c2979e3a8fec3327dfa64983b4f61c28: Waiting for close lock at 1731596060764Disabling compacts and flushes for region at 1731596060764Disabling writes for close at 1731596060764Writing region close event to WAL at 1731596060764Closed at 1731596060764 2024-11-14T14:54:20,766 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T14:54:20,767 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731596060766"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731596060766"}]},"ts":"1731596060766"} 2024-11-14T14:54:20,773 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T14:54:20,775 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T14:54:20,775 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731596060775"}]},"ts":"1731596060775"} 2024-11-14T14:54:20,778 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta 2024-11-14T14:54:20,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=c2979e3a8fec3327dfa64983b4f61c28, ASSIGN}] 2024-11-14T14:54:20,784 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=c2979e3a8fec3327dfa64983b4f61c28, ASSIGN 2024-11-14T14:54:20,786 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=c2979e3a8fec3327dfa64983b4f61c28, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,40527,1731596059482; forceNewPlan=false, retain=false 2024-11-14T14:54:20,810 INFO [RS:1;da1c6afcd1f9:34075 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C34075%2C1731596060544, suffix=, logDir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544, archiveDir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/oldWALs, maxLogs=32 2024-11-14T14:54:20,811 INFO [RS:1;da1c6afcd1f9:34075 {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 2024-11-14T14:54:20,821 INFO [RS:1;da1c6afcd1f9:34075 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 2024-11-14T14:54:20,833 DEBUG [RS:1;da1c6afcd1f9:34075 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40049:40049),(127.0.0.1/127.0.0.1:38019:38019)] 2024-11-14T14:54:20,937 INFO [da1c6afcd1f9:40173 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T14:54:20,938 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c2979e3a8fec3327dfa64983b4f61c28, regionState=OPENING, regionLocation=da1c6afcd1f9,40527,1731596059482 2024-11-14T14:54:20,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=c2979e3a8fec3327dfa64983b4f61c28, ASSIGN because future has completed 2024-11-14T14:54:20,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2979e3a8fec3327dfa64983b4f61c28, server=da1c6afcd1f9,40527,1731596059482}] 2024-11-14T14:54:21,100 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28. 2024-11-14T14:54:21,100 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c2979e3a8fec3327dfa64983b4f61c28, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:54:21,100 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:21,101 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:54:21,101 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:21,101 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:21,102 INFO [StoreOpener-c2979e3a8fec3327dfa64983b4f61c28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:21,104 INFO [StoreOpener-c2979e3a8fec3327dfa64983b4f61c28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2979e3a8fec3327dfa64983b4f61c28 columnFamilyName info 2024-11-14T14:54:21,104 DEBUG [StoreOpener-c2979e3a8fec3327dfa64983b4f61c28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:54:21,104 INFO [StoreOpener-c2979e3a8fec3327dfa64983b4f61c28-1 {}] regionserver.HStore(327): Store=c2979e3a8fec3327dfa64983b4f61c28/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:54:21,104 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:21,105 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:21,105 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:21,106 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:21,106 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:21,107 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:21,109 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:54:21,110 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c2979e3a8fec3327dfa64983b4f61c28; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=810700, jitterRate=0.03085930645465851}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:54:21,110 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:21,111 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c2979e3a8fec3327dfa64983b4f61c28: Running coprocessor pre-open hook at 1731596061101Writing region info on filesystem at 1731596061101Initializing all the Stores at 1731596061102 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596061102Cleaning up temporary data from old regions at 1731596061106 (+4 ms)Running coprocessor post-open hooks at 1731596061110 (+4 ms)Region opened successfully at 1731596061110 2024-11-14T14:54:21,112 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28., pid=6, masterSystemTime=1731596061095 2024-11-14T14:54:21,114 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28. 2024-11-14T14:54:21,114 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28. 2024-11-14T14:54:21,115 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c2979e3a8fec3327dfa64983b4f61c28, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,40527,1731596059482 2024-11-14T14:54:21,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2979e3a8fec3327dfa64983b4f61c28, server=da1c6afcd1f9,40527,1731596059482 because future has completed 2024-11-14T14:54:21,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T14:54:21,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c2979e3a8fec3327dfa64983b4f61c28, server=da1c6afcd1f9,40527,1731596059482 in 178 msec 2024-11-14T14:54:21,124 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T14:54:21,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=c2979e3a8fec3327dfa64983b4f61c28, ASSIGN in 343 msec 2024-11-14T14:54:21,126 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T14:54:21,126 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731596061126"}]},"ts":"1731596061126"} 2024-11-14T14:54:21,128 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta 2024-11-14T14:54:21,129 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T14:54:21,131 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 417 msec 2024-11-14T14:54:25,906 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-14T14:54:25,909 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:54:25,923 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:54:25,925 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:54:25,926 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:54:25,933 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath' 2024-11-14T14:54:29,616 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T14:54:29,617 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T14:54:29,617 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-11-14T14:54:29,617 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer 2024-11-14T14:54:29,618 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:54:29,618 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T14:54:29,618 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T14:54:29,618 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T14:54:30,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40173 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:54:30,728 INFO [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed 2024-11-14T14:54:30,728 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100 2024-11-14T14:54:30,731 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath 2024-11-14T14:54:30,731 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28. 2024-11-14T14:54:30,745 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:54:30,748 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:54:30,749 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:54:30,749 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:54:30,749 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:54:30,749 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ff5703b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:54:30,750 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@36d0b5ff{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:54:30,864 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@9038e26{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/java.io.tmpdir/jetty-localhost-37339-hadoop-hdfs-3_4_1-tests_jar-_-any-8978804807251710902/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:54:30,864 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@501a5826{HTTP/1.1, (http/1.1)}{localhost:37339} 2024-11-14T14:54:30,865 INFO [Time-limited test {}] server.Server(415): Started @116486ms 2024-11-14T14:54:30,866 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:54:30,904 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:54:30,907 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:54:30,911 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:54:30,911 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:54:30,911 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:54:30,912 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@167a7fde{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:54:30,912 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@712f5f14{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:54:30,952 WARN [Thread-830 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data6/current/BP-1040518630-172.17.0.2-1731596058577/current, will proceed with Du for space computation calculation, 2024-11-14T14:54:30,952 WARN [Thread-829 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data5/current/BP-1040518630-172.17.0.2-1731596058577/current, will proceed with Du for space computation calculation, 2024-11-14T14:54:30,969 WARN [Thread-809 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:54:30,971 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc3488ac417345f9c with lease ID 0x2ad170a756447204: Processing first storage report for DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02 from datanode DatanodeRegistration(127.0.0.1:45899, datanodeUuid=1852e955-c282-4a6e-9346-3f9f23284655, infoPort=39223, infoSecurePort=0, ipcPort=40199, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577) 2024-11-14T14:54:30,971 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc3488ac417345f9c with lease ID 0x2ad170a756447204: from storage DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02 node DatanodeRegistration(127.0.0.1:45899, datanodeUuid=1852e955-c282-4a6e-9346-3f9f23284655, infoPort=39223, infoSecurePort=0, ipcPort=40199, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:54:30,971 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc3488ac417345f9c with lease ID 0x2ad170a756447204: Processing first storage report for DS-64e649ca-818a-4944-87ba-b98092dabeaa from datanode DatanodeRegistration(127.0.0.1:45899, datanodeUuid=1852e955-c282-4a6e-9346-3f9f23284655, infoPort=39223, infoSecurePort=0, ipcPort=40199, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577) 2024-11-14T14:54:30,971 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc3488ac417345f9c with lease ID 0x2ad170a756447204: from storage DS-64e649ca-818a-4944-87ba-b98092dabeaa node DatanodeRegistration(127.0.0.1:45899, datanodeUuid=1852e955-c282-4a6e-9346-3f9f23284655, infoPort=39223, infoSecurePort=0, ipcPort=40199, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:54:31,030 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@272698f5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/java.io.tmpdir/jetty-localhost-41473-hadoop-hdfs-3_4_1-tests_jar-_-any-13877260466366888204/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:54:31,031 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@15bbd738{HTTP/1.1, (http/1.1)}{localhost:41473} 2024-11-14T14:54:31,031 INFO [Time-limited test {}] server.Server(415): Started @116653ms 2024-11-14T14:54:31,032 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:54:31,066 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:54:31,070 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:54:31,071 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:54:31,071 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:54:31,072 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:54:31,072 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1dc59954{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:54:31,073 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@68004957{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:54:31,127 WARN [Thread-864 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data7/current/BP-1040518630-172.17.0.2-1731596058577/current, will proceed with Du for space computation calculation, 2024-11-14T14:54:31,127 WARN [Thread-865 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data8/current/BP-1040518630-172.17.0.2-1731596058577/current, will proceed with Du for space computation calculation, 2024-11-14T14:54:31,144 WARN [Thread-844 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:54:31,147 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3d28fa2b92ffb379 with lease ID 0x2ad170a756447205: Processing first storage report for DS-ab0c677b-1249-4dbd-a7c5-65698bee7202 from datanode DatanodeRegistration(127.0.0.1:40217, datanodeUuid=8436098f-b3ac-48c2-9a57-e18f2bb1c609, infoPort=37815, infoSecurePort=0, ipcPort=37021, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577) 2024-11-14T14:54:31,147 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3d28fa2b92ffb379 with lease ID 0x2ad170a756447205: from storage DS-ab0c677b-1249-4dbd-a7c5-65698bee7202 node DatanodeRegistration(127.0.0.1:40217, datanodeUuid=8436098f-b3ac-48c2-9a57-e18f2bb1c609, infoPort=37815, infoSecurePort=0, ipcPort=37021, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:54:31,147 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3d28fa2b92ffb379 with lease ID 0x2ad170a756447205: Processing first storage report for DS-62bb9f09-768e-4905-b2e6-6abe1091950b from datanode DatanodeRegistration(127.0.0.1:40217, datanodeUuid=8436098f-b3ac-48c2-9a57-e18f2bb1c609, infoPort=37815, infoSecurePort=0, ipcPort=37021, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577) 2024-11-14T14:54:31,147 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3d28fa2b92ffb379 with lease ID 0x2ad170a756447205: from storage DS-62bb9f09-768e-4905-b2e6-6abe1091950b node DatanodeRegistration(127.0.0.1:40217, datanodeUuid=8436098f-b3ac-48c2-9a57-e18f2bb1c609, infoPort=37815, infoSecurePort=0, ipcPort=37021, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:54:31,201 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@28246fba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/java.io.tmpdir/jetty-localhost-34395-hadoop-hdfs-3_4_1-tests_jar-_-any-17026135805835724435/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:54:31,201 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5056747b{HTTP/1.1, (http/1.1)}{localhost:34395} 2024-11-14T14:54:31,202 INFO [Time-limited test {}] server.Server(415): Started @116823ms 2024-11-14T14:54:31,203 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:54:31,301 WARN [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data9/current/BP-1040518630-172.17.0.2-1731596058577/current, will proceed with Du for space computation calculation, 2024-11-14T14:54:31,301 WARN [Thread-891 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data10/current/BP-1040518630-172.17.0.2-1731596058577/current, will proceed with Du for space computation calculation, 2024-11-14T14:54:31,318 WARN [Thread-879 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:54:31,321 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x652a1f4a256f5c8f with lease ID 0x2ad170a756447206: Processing first storage report for DS-971a76db-9204-4270-ba46-5b1009f5de99 from datanode DatanodeRegistration(127.0.0.1:44787, datanodeUuid=3961c212-814b-45b5-8059-b2afc9c559ae, infoPort=34017, infoSecurePort=0, ipcPort=41381, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577) 2024-11-14T14:54:31,321 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x652a1f4a256f5c8f with lease ID 0x2ad170a756447206: from storage DS-971a76db-9204-4270-ba46-5b1009f5de99 node DatanodeRegistration(127.0.0.1:44787, datanodeUuid=3961c212-814b-45b5-8059-b2afc9c559ae, infoPort=34017, infoSecurePort=0, ipcPort=41381, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:54:31,321 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x652a1f4a256f5c8f with lease ID 0x2ad170a756447206: Processing first storage report for DS-c154e900-a66e-46d7-b32e-3488da922212 from datanode DatanodeRegistration(127.0.0.1:44787, datanodeUuid=3961c212-814b-45b5-8059-b2afc9c559ae, infoPort=34017, infoSecurePort=0, ipcPort=41381, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577) 2024-11-14T14:54:31,321 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x652a1f4a256f5c8f with lease ID 0x2ad170a756447206: from storage DS-c154e900-a66e-46d7-b32e-3488da922212 node DatanodeRegistration(127.0.0.1:44787, datanodeUuid=3961c212-814b-45b5-8059-b2afc9c559ae, infoPort=34017, infoSecurePort=0, ipcPort=41381, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:54:31,324 WARN [ResponseProcessor for block BP-1040518630-172.17.0.2-1731596058577:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1040518630-172.17.0.2-1731596058577:blk_1073741834_1010 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:31,324 WARN [ResponseProcessor for block BP-1040518630-172.17.0.2-1731596058577:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1040518630-172.17.0.2-1731596058577:blk_1073741833_1009 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:31,324 WARN [ResponseProcessor for block BP-1040518630-172.17.0.2-1731596058577:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1040518630-172.17.0.2-1731596058577:blk_1073741837_1013 java.io.IOException: Bad response ERROR for BP-1040518630-172.17.0.2-1731596058577:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:31,324 WARN [DataStreamer for file /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta block BP-1040518630-172.17.0.2-1731596058577:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK], DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]) is bad. 2024-11-14T14:54:31,325 WARN [DataStreamer for file /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 block BP-1040518630-172.17.0.2-1731596058577:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK], DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]) is bad. 2024-11-14T14:54:31,325 WARN [ResponseProcessor for block BP-1040518630-172.17.0.2-1731596058577:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1040518630-172.17.0.2-1731596058577:blk_1073741830_1006 java.io.IOException: Bad response ERROR for BP-1040518630-172.17.0.2-1731596058577:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:31,325 WARN [DataStreamer for file /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596060015 block BP-1040518630-172.17.0.2-1731596058577:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK], DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]) is bad. 2024-11-14T14:54:31,325 WARN [DataStreamer for file /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/WALs/da1c6afcd1f9,40173,1731596059405/da1c6afcd1f9%2C40173%2C1731596059405.1731596059635 block BP-1040518630-172.17.0.2-1731596058577:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK], DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]) is bad. 2024-11-14T14:54:31,325 WARN [PacketResponder: BP-1040518630-172.17.0.2-1731596058577:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:35719] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:31,325 WARN [PacketResponder: BP-1040518630-172.17.0.2-1731596058577:blk_1073741837_1013, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:35719] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Broken pipe at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:31,326 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:50634 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:35719:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50634 dst: /127.0.0.1:35719 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:31,326 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:40424 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:33793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40424 dst: /127.0.0.1:33793 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:31,326 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:50630 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:35719:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50630 dst: /127.0.0.1:35719 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:31,326 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1736656620_22 at /127.0.0.1:57880 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:33793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57880 dst: /127.0.0.1:33793 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:31,326 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1736656620_22 at /127.0.0.1:59412 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:35719:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59412 dst: /127.0.0.1:35719 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:31,326 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-207604970_22 at /127.0.0.1:50664 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:35719:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50664 dst: /127.0.0.1:35719 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:31,327 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:40410 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:33793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40410 dst: /127.0.0.1:33793 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:31,328 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@f7d5275{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:54:31,328 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-207604970_22 at /127.0.0.1:40456 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:33793:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40456 dst: /127.0.0.1:33793 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:31,328 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5f783084{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:54:31,329 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:54:31,329 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69bbaec1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:54:31,329 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1ec1a06e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/hadoop.log.dir/,STOPPED} 2024-11-14T14:54:31,331 WARN [BP-1040518630-172.17.0.2-1731596058577 heartbeating to localhost/127.0.0.1:35595 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:54:31,331 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:54:31,331 WARN [BP-1040518630-172.17.0.2-1731596058577 heartbeating to localhost/127.0.0.1:35595 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1040518630-172.17.0.2-1731596058577 (Datanode Uuid ed0576a8-2ece-4054-b155-ea617541c969) service to localhost/127.0.0.1:35595 2024-11-14T14:54:31,331 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:54:31,331 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data3/current/BP-1040518630-172.17.0.2-1731596058577 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:54:31,332 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data4/current/BP-1040518630-172.17.0.2-1731596058577 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:54:31,332 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:54:31,332 WARN [DataStreamer for file /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/WALs/da1c6afcd1f9,40173,1731596059405/da1c6afcd1f9%2C40173%2C1731596059405.1731596059635 block BP-1040518630-172.17.0.2-1731596058577:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:31,333 WARN [DataStreamer for file /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596060015 block BP-1040518630-172.17.0.2-1731596058577:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:31,333 WARN [DataStreamer for file /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta block BP-1040518630-172.17.0.2-1731596058577:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:31,333 ERROR [org.apache.hadoop.hdfs.server.datanode.DataXceiver@97136ad {}] datanode.DataXceiver(331): 127.0.0.1:33793:DataXceiver error processing unknown operation src: /127.0.0.1:33680 dst: /127.0.0.1:33793 java.io.IOException: Server closed. at org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.addPeer(DataXceiverServer.java:334) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:232) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:31,334 WARN [DataStreamer for file /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 block BP-1040518630-172.17.0.2-1731596058577:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013 java.net.SocketException: Connection reset at sun.nio.ch.SocketChannelImpl.throwConnectionReset(SocketChannelImpl.java:394) ~[?:?] at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:426) ~[?:?] at org.apache.hadoop.net.SocketInputStream$Reader.performIO(SocketInputStream.java:57) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) ~[hadoop-common-3.4.1.jar:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at java.io.FilterInputStream.read(FilterInputStream.java:82) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:527) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1931) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:31,337 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@692b8c40{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:54:31,338 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1369ef4b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:54:31,338 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:54:31,338 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@257cf4bb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:54:31,338 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48743db4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/hadoop.log.dir/,STOPPED} 2024-11-14T14:54:31,339 WARN [BP-1040518630-172.17.0.2-1731596058577 heartbeating to localhost/127.0.0.1:35595 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:54:31,339 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:54:31,339 WARN [BP-1040518630-172.17.0.2-1731596058577 heartbeating to localhost/127.0.0.1:35595 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1040518630-172.17.0.2-1731596058577 (Datanode Uuid 0ba76aad-cfa9-4a36-afa7-cde9f51172e1) service to localhost/127.0.0.1:35595 2024-11-14T14:54:31,339 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:54:31,340 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data1/current/BP-1040518630-172.17.0.2-1731596058577 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:54:31,340 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data2/current/BP-1040518630-172.17.0.2-1731596058577 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:54:31,340 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:54:31,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28., hostname=da1c6afcd1f9,40527,1731596059482, seqNum=2] 2024-11-14T14:54:31,346 ERROR [FSHLog-0-hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7-prefix:da1c6afcd1f9,40527,1731596059482 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:31,346 WARN [FSHLog-0-hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7-prefix:da1c6afcd1f9,40527,1731596059482 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:31,346 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:31,346 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog da1c6afcd1f9%2C40527%2C1731596059482:(num 1731596060015) roll requested 2024-11-14T14:54:31,347 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C40527%2C1731596059482.1731596071346 2024-11-14T14:54:31,349 WARN [Thread-902 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741838_1018 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:31,350 WARN [Thread-902 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK], DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]) is bad. 2024-11-14T14:54:31,350 WARN [Thread-902 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741838_1018 2024-11-14T14:54:31,352 WARN [Thread-902 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK] 2024-11-14T14:54:31,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:31,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:31,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:31,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:31,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:31,359 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596060015 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596071346 2024-11-14T14:54:31,359 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:31,359 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:31,359 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37815:37815),(127.0.0.1/127.0.0.1:39223:39223)] 2024-11-14T14:54:31,359 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596060015 is not closed yet, will try archiving it next time 2024-11-14T14:54:31,360 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-14T14:54:31,360 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-14T14:54:31,361 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596060015 2024-11-14T14:54:31,363 WARN [IPC Server handler 1 on default port 35595 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596060015 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741833_1009 2024-11-14T14:54:31,366 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596060015 after 4ms 2024-11-14T14:54:31,590 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:32,649 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:33,360 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:33,361 INFO [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596071346 2024-11-14T14:54:33,362 WARN [ResponseProcessor for block BP-1040518630-172.17.0.2-1731596058577:blk_1073741839_1019 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1040518630-172.17.0.2-1731596058577:blk_1073741839_1019 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:33,362 WARN [DataStreamer for file /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596071346 block BP-1040518630-172.17.0.2-1731596058577:blk_1073741839_1019 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741839_1019 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK], DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]) is bad. 2024-11-14T14:54:33,363 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:53484 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:40217:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53484 dst: /127.0.0.1:40217 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:33,363 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:49062 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741839_1019] {}] datanode.DataXceiver(331): 127.0.0.1:45899:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49062 dst: /127.0.0.1:45899 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:33,366 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@272698f5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:54:33,366 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@15bbd738{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:54:33,366 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:54:33,367 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@712f5f14{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:54:33,367 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@167a7fde{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/hadoop.log.dir/,STOPPED} 2024-11-14T14:54:33,369 WARN [BP-1040518630-172.17.0.2-1731596058577 heartbeating to localhost/127.0.0.1:35595 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:54:33,369 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:54:33,369 WARN [BP-1040518630-172.17.0.2-1731596058577 heartbeating to localhost/127.0.0.1:35595 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1040518630-172.17.0.2-1731596058577 (Datanode Uuid 8436098f-b3ac-48c2-9a57-e18f2bb1c609) service to localhost/127.0.0.1:35595 2024-11-14T14:54:33,369 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:54:33,369 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data7/current/BP-1040518630-172.17.0.2-1731596058577 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:54:33,369 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data8/current/BP-1040518630-172.17.0.2-1731596058577 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:54:33,370 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:54:33,590 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:34,649 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:35,361 WARN [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]] 2024-11-14T14:54:35,361 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:35,361 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog da1c6afcd1f9%2C40527%2C1731596059482:(num 1731596071346) roll requested 2024-11-14T14:54:35,361 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C40527%2C1731596059482.1731596075361 2024-11-14T14:54:35,364 WARN [Thread-911 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:35,365 WARN [Thread-911 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK], DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]) is bad. 2024-11-14T14:54:35,365 WARN [Thread-911 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741840_1022 2024-11-14T14:54:35,365 WARN [Thread-911 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK] 2024-11-14T14:54:35,367 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596060015 after 4006ms 2024-11-14T14:54:35,368 WARN [Thread-911 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1023 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40217 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:35,368 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54556 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741841_1023] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data10]'}, localName='127.0.0.1:44787', datanodeUuid='3961c212-814b-45b5-8059-b2afc9c559ae', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741841_1023 to mirror 127.0.0.1:40217 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:35,368 WARN [Thread-911 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK], DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]) is bad. 2024-11-14T14:54:35,368 WARN [Thread-911 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741841_1023 2024-11-14T14:54:35,368 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54556 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741841_1023] {}] datanode.BlockReceiver(316): Block 1073741841 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-14T14:54:35,368 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54556 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:44787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54556 dst: /127.0.0.1:44787 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:35,368 WARN [Thread-911 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK] 2024-11-14T14:54:35,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:35,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:35,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:35,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:35,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:35,373 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596071346 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596075361 2024-11-14T14:54:35,375 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T14:54:35,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45899 is added to blk_1073741839_1021 (size=2431) 2024-11-14T14:54:35,376 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34017:34017),(127.0.0.1/127.0.0.1:39223:39223)] 2024-11-14T14:54:35,376 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596060015 is not closed yet, will try archiving it next time 2024-11-14T14:54:35,376 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596071346 is not closed yet, will try archiving it next time 2024-11-14T14:54:35,590 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:35,776 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596060015 is not closed yet, will try archiving it next time 2024-11-14T14:54:36,649 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:36,983 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@122fa983[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45899, datanodeUuid=1852e955-c282-4a6e-9346-3f9f23284655, infoPort=39223, infoSecurePort=0, ipcPort=40199, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577):Failed to transfer BP-1040518630-172.17.0.2-1731596058577:blk_1073741839_1021 to 127.0.0.1:33793 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:37,377 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:37,378 WARN [ResponseProcessor for block BP-1040518630-172.17.0.2-1731596058577:blk_1073741842_1024 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1040518630-172.17.0.2-1731596058577:blk_1073741842_1024 java.io.IOException: Bad response ERROR for BP-1040518630-172.17.0.2-1731596058577:blk_1073741842_1024 from datanode DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:37,379 WARN [DataStreamer for file /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596075361 block BP-1040518630-172.17.0.2-1731596058577:blk_1073741842_1024 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741842_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK], DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]) is bad. 2024-11-14T14:54:37,379 WARN [PacketResponder: BP-1040518630-172.17.0.2-1731596058577:blk_1073741842_1024, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:45899] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:37,379 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54568 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:44787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54568 dst: /127.0.0.1:44787 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:37,380 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:49078 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741842_1024] {}] datanode.DataXceiver(331): 127.0.0.1:45899:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49078 dst: /127.0.0.1:45899 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:37,381 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@9038e26{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:54:37,381 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@501a5826{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:54:37,381 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:54:37,381 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@36d0b5ff{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:54:37,382 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ff5703b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/hadoop.log.dir/,STOPPED} 2024-11-14T14:54:37,383 WARN [BP-1040518630-172.17.0.2-1731596058577 heartbeating to localhost/127.0.0.1:35595 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:54:37,383 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:54:37,383 WARN [BP-1040518630-172.17.0.2-1731596058577 heartbeating to localhost/127.0.0.1:35595 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1040518630-172.17.0.2-1731596058577 (Datanode Uuid 1852e955-c282-4a6e-9346-3f9f23284655) service to localhost/127.0.0.1:35595 2024-11-14T14:54:37,383 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:54:37,384 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data5/current/BP-1040518630-172.17.0.2-1731596058577 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:54:37,384 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data6/current/BP-1040518630-172.17.0.2-1731596058577 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:54:37,384 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:54:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40527 {}] regionserver.HRegion(8855): Flush requested on c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:37,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2979e3a8fec3327dfa64983b4f61c28 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-14T14:54:37,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/14bf962eb0c8455b8bf68c0fbbc1a5ec is 1080, key is row0002/info:/1731596073371/Put/seqid=0 2024-11-14T14:54:37,417 WARN [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:37,417 WARN [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK], DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]) is bad. 2024-11-14T14:54:37,417 WARN [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741843_1026 2024-11-14T14:54:37,418 WARN [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK] 2024-11-14T14:54:37,419 WARN [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:37,419 WARN [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK], DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]) is bad. 2024-11-14T14:54:37,419 WARN [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741844_1027 2024-11-14T14:54:37,419 WARN [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK] 2024-11-14T14:54:37,421 WARN [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:37,421 WARN [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK], DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]) is bad. 2024-11-14T14:54:37,421 WARN [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741845_1028 2024-11-14T14:54:37,422 WARN [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK] 2024-11-14T14:54:37,424 WARN [Thread-923 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33793 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:37,424 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54586 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741846_1029] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data10]'}, localName='127.0.0.1:44787', datanodeUuid='3961c212-814b-45b5-8059-b2afc9c559ae', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741846_1029 to mirror 127.0.0.1:33793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:37,424 WARN [Thread-923 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK], DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]) is bad. 2024-11-14T14:54:37,424 WARN [Thread-923 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741846_1029 2024-11-14T14:54:37,424 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54586 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741846_1029] {}] datanode.BlockReceiver(316): Block 1073741846 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-14T14:54:37,424 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54586 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741846_1029] {}] datanode.DataXceiver(331): 127.0.0.1:44787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54586 dst: /127.0.0.1:44787 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:37,424 WARN [Thread-923 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK] 2024-11-14T14:54:37,425 WARN [IPC Server handler 3 on default port 35595 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-14T14:54:37,425 WARN [IPC Server handler 3 on default port 35595 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-14T14:54:37,425 WARN [IPC Server handler 3 on default port 35595 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-14T14:54:37,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741847_1030 (size=10347) 2024-11-14T14:54:37,591 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:37,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/14bf962eb0c8455b8bf68c0fbbc1a5ec 2024-11-14T14:54:37,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/14bf962eb0c8455b8bf68c0fbbc1a5ec as hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/14bf962eb0c8455b8bf68c0fbbc1a5ec 2024-11-14T14:54:37,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/14bf962eb0c8455b8bf68c0fbbc1a5ec, entries=5, sequenceid=11, filesize=10.1 K 2024-11-14T14:54:37,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for c2979e3a8fec3327dfa64983b4f61c28 in 455ms, sequenceid=11, compaction requested=false 2024-11-14T14:54:37,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2979e3a8fec3327dfa64983b4f61c28: 2024-11-14T14:54:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40527 {}] regionserver.HRegion(8855): Flush requested on c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:38,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2979e3a8fec3327dfa64983b4f61c28 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB 2024-11-14T14:54:38,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/63a940bb12f446fb9fcf3d733d803b1a is 1080, key is row0007/info:/1731596077395/Put/seqid=0 2024-11-14T14:54:38,025 WARN [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:38,025 WARN [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK], DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]) is bad. 2024-11-14T14:54:38,025 WARN [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741848_1031 2024-11-14T14:54:38,025 WARN [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK] 2024-11-14T14:54:38,027 WARN [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:38,027 WARN [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK], DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]) is bad. 2024-11-14T14:54:38,027 WARN [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741849_1032 2024-11-14T14:54:38,027 WARN [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK] 2024-11-14T14:54:38,028 WARN [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:38,029 WARN [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK], DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]) is bad. 2024-11-14T14:54:38,029 WARN [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741850_1033 2024-11-14T14:54:38,029 WARN [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK] 2024-11-14T14:54:38,031 WARN [Thread-929 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33793 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:38,031 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54616 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741851_1034] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data10]'}, localName='127.0.0.1:44787', datanodeUuid='3961c212-814b-45b5-8059-b2afc9c559ae', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741851_1034 to mirror 127.0.0.1:33793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:38,031 WARN [Thread-929 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK], DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]) is bad. 2024-11-14T14:54:38,031 WARN [Thread-929 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741851_1034 2024-11-14T14:54:38,032 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54616 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741851_1034] {}] datanode.BlockReceiver(316): Block 1073741851 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-14T14:54:38,032 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54616 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741851_1034] {}] datanode.DataXceiver(331): 127.0.0.1:44787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54616 dst: /127.0.0.1:44787 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:38,032 WARN [Thread-929 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK] 2024-11-14T14:54:38,033 WARN [IPC Server handler 3 on default port 35595 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-14T14:54:38,033 WARN [IPC Server handler 3 on default port 35595 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-14T14:54:38,033 WARN [IPC Server handler 3 on default port 35595 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-14T14:54:38,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741852_1035 (size=12506) 2024-11-14T14:54:38,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/63a940bb12f446fb9fcf3d733d803b1a 2024-11-14T14:54:38,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/63a940bb12f446fb9fcf3d733d803b1a as hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/63a940bb12f446fb9fcf3d733d803b1a 2024-11-14T14:54:38,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/63a940bb12f446fb9fcf3d733d803b1a, entries=7, sequenceid=24, filesize=12.2 K 2024-11-14T14:54:38,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for c2979e3a8fec3327dfa64983b4f61c28 in 432ms, sequenceid=24, compaction requested=false 2024-11-14T14:54:38,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2979e3a8fec3327dfa64983b4f61c28: 2024-11-14T14:54:38,451 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K 2024-11-14T14:54:38,451 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:54:38,451 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/63a940bb12f446fb9fcf3d733d803b1a because midkey is the same as first or last row 2024-11-14T14:54:38,650 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:39,377 WARN [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK]] 2024-11-14T14:54:39,377 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:39,377 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog da1c6afcd1f9%2C40527%2C1731596059482:(num 1731596075361) roll requested 2024-11-14T14:54:39,378 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C40527%2C1731596059482.1731596079378 2024-11-14T14:54:39,381 WARN [Thread-934 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:39,381 WARN [Thread-934 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK], DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]) is bad. 2024-11-14T14:54:39,381 WARN [Thread-934 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741853_1036 2024-11-14T14:54:39,382 WARN [Thread-934 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK] 2024-11-14T14:54:39,383 WARN [Thread-934 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:39,383 WARN [Thread-934 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK], DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]) is bad. 2024-11-14T14:54:39,383 WARN [Thread-934 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741854_1037 2024-11-14T14:54:39,383 WARN [Thread-934 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK] 2024-11-14T14:54:39,386 WARN [Thread-934 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33793 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:39,386 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54628 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741855_1038] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data10]'}, localName='127.0.0.1:44787', datanodeUuid='3961c212-814b-45b5-8059-b2afc9c559ae', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741855_1038 to mirror 127.0.0.1:33793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:39,386 WARN [Thread-934 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK], DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]) is bad. 2024-11-14T14:54:39,386 WARN [Thread-934 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741855_1038 2024-11-14T14:54:39,386 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54628 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741855_1038] {}] datanode.BlockReceiver(316): Block 1073741855 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-14T14:54:39,386 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54628 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741855_1038] {}] datanode.DataXceiver(331): 127.0.0.1:44787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54628 dst: /127.0.0.1:44787 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:39,386 WARN [Thread-934 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK] 2024-11-14T14:54:39,388 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54634 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741856_1039] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data10]'}, localName='127.0.0.1:44787', datanodeUuid='3961c212-814b-45b5-8059-b2afc9c559ae', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741856_1039 to mirror 127.0.0.1:35719 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:39,388 WARN [Thread-934 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35719 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:39,388 WARN [Thread-934 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK], DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]) is bad. 2024-11-14T14:54:39,388 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54634 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741856_1039] {}] datanode.BlockReceiver(316): Block 1073741856 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-14T14:54:39,388 WARN [Thread-934 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741856_1039 2024-11-14T14:54:39,389 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54634 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741856_1039] {}] datanode.DataXceiver(331): 127.0.0.1:44787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54634 dst: /127.0.0.1:44787 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:39,389 WARN [Thread-934 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK] 2024-11-14T14:54:39,390 WARN [IPC Server handler 0 on default port 35595 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-14T14:54:39,390 WARN [IPC Server handler 0 on default port 35595 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-14T14:54:39,390 WARN [IPC Server handler 0 on default port 35595 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-14T14:54:39,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:39,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:39,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:39,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:39,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:39,393 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596075361 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596079378 2024-11-14T14:54:39,394 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34017:34017)] 2024-11-14T14:54:39,394 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596060015 is not closed yet, will try archiving it next time 2024-11-14T14:54:39,394 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596075361 is not closed yet, will try archiving it next time 2024-11-14T14:54:39,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741842_1025 (size=25992) 2024-11-14T14:54:39,394 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596071346 to hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/oldWALs/da1c6afcd1f9%2C40527%2C1731596059482.1731596071346 2024-11-14T14:54:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40527 {}] regionserver.HRegion(8855): Flush requested on c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:39,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2979e3a8fec3327dfa64983b4f61c28 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-14T14:54:39,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/1d1f7cb7b6144f51ac1d9933aef40d45 is 1079, key is tmprow/info:/1731596079435/Put/seqid=0 2024-11-14T14:54:39,443 WARN [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:39,443 WARN [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK], DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]) is bad. 2024-11-14T14:54:39,443 WARN [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741858_1041 2024-11-14T14:54:39,443 WARN [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK] 2024-11-14T14:54:39,445 WARN [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:39,445 WARN [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK], DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]) is bad. 2024-11-14T14:54:39,445 WARN [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741859_1042 2024-11-14T14:54:39,445 WARN [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK] 2024-11-14T14:54:39,446 WARN [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:39,446 WARN [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK], DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]) is bad. 2024-11-14T14:54:39,446 WARN [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741860_1043 2024-11-14T14:54:39,447 WARN [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK] 2024-11-14T14:54:39,448 WARN [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:39,448 WARN [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK], DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]) is bad. 2024-11-14T14:54:39,448 WARN [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741861_1044 2024-11-14T14:54:39,449 WARN [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK] 2024-11-14T14:54:39,449 WARN [IPC Server handler 3 on default port 35595 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-14T14:54:39,449 WARN [IPC Server handler 3 on default port 35595 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-14T14:54:39,449 WARN [IPC Server handler 3 on default port 35595 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-14T14:54:39,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741862_1045 (size=6027) 2024-11-14T14:54:39,591 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:39,795 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596060015 is not closed yet, will try archiving it next time 2024-11-14T14:54:39,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/1d1f7cb7b6144f51ac1d9933aef40d45 2024-11-14T14:54:39,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/1d1f7cb7b6144f51ac1d9933aef40d45 as hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/1d1f7cb7b6144f51ac1d9933aef40d45 2024-11-14T14:54:39,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/1d1f7cb7b6144f51ac1d9933aef40d45, entries=1, sequenceid=34, filesize=5.9 K 2024-11-14T14:54:39,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for c2979e3a8fec3327dfa64983b4f61c28 in 430ms, sequenceid=34, compaction requested=true 2024-11-14T14:54:39,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2979e3a8fec3327dfa64983b4f61c28: 2024-11-14T14:54:39,866 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K 2024-11-14T14:54:39,866 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:54:39,866 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/63a940bb12f446fb9fcf3d733d803b1a because midkey is the same as first or last row 2024-11-14T14:54:39,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2979e3a8fec3327dfa64983b4f61c28:info, priority=-2147483648, current under compaction store size is 1 2024-11-14T14:54:39,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:54:39,867 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T14:54:39,868 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T14:54:39,868 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.HStore(1541): c2979e3a8fec3327dfa64983b4f61c28/info is initiating minor compaction (all files) 2024-11-14T14:54:39,868 INFO [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2979e3a8fec3327dfa64983b4f61c28/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28. 2024-11-14T14:54:39,868 INFO [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/14bf962eb0c8455b8bf68c0fbbc1a5ec, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/63a940bb12f446fb9fcf3d733d803b1a, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/1d1f7cb7b6144f51ac1d9933aef40d45] into tmpdir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp, totalSize=28.2 K 2024-11-14T14:54:39,869 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14bf962eb0c8455b8bf68c0fbbc1a5ec, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731596073371 2024-11-14T14:54:39,870 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63a940bb12f446fb9fcf3d733d803b1a, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1731596077395 2024-11-14T14:54:39,870 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d1f7cb7b6144f51ac1d9933aef40d45, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731596079435 2024-11-14T14:54:39,883 INFO [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2979e3a8fec3327dfa64983b4f61c28#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T14:54:39,884 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/ec29e5b7b8574cebb9237b1ca52ef126 is 1080, key is row0002/info:/1731596073371/Put/seqid=0 2024-11-14T14:54:39,886 WARN [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:39,886 WARN [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK], DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]) is bad. 2024-11-14T14:54:39,886 WARN [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741863_1046 2024-11-14T14:54:39,886 WARN [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK] 2024-11-14T14:54:39,887 WARN [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:39,887 WARN [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK], DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]) is bad. 2024-11-14T14:54:39,888 WARN [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741864_1047 2024-11-14T14:54:39,888 WARN [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK] 2024-11-14T14:54:39,890 WARN [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45899 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:39,890 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37300 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741865_1048] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data10]'}, localName='127.0.0.1:44787', datanodeUuid='3961c212-814b-45b5-8059-b2afc9c559ae', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741865_1048 to mirror 127.0.0.1:45899 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:39,890 WARN [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK], DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]) is bad. 2024-11-14T14:54:39,890 WARN [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741865_1048 2024-11-14T14:54:39,890 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37300 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741865_1048] {}] datanode.BlockReceiver(316): Block 1073741865 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-14T14:54:39,891 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37300 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741865_1048] {}] datanode.DataXceiver(331): 127.0.0.1:44787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37300 dst: /127.0.0.1:44787 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:39,891 WARN [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK] 2024-11-14T14:54:39,893 WARN [Thread-946 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33793 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:39,893 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37312 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741866_1049] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data10]'}, localName='127.0.0.1:44787', datanodeUuid='3961c212-814b-45b5-8059-b2afc9c559ae', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741866_1049 to mirror 127.0.0.1:33793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:39,893 WARN [Thread-946 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK], DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]) is bad. 2024-11-14T14:54:39,893 WARN [Thread-946 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741866_1049 2024-11-14T14:54:39,893 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37312 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741866_1049] {}] datanode.BlockReceiver(316): Block 1073741866 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-14T14:54:39,893 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37312 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741866_1049] {}] datanode.DataXceiver(331): 127.0.0.1:44787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37312 dst: /127.0.0.1:44787 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:39,894 WARN [Thread-946 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK] 2024-11-14T14:54:39,894 WARN [IPC Server handler 4 on default port 35595 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-14T14:54:39,894 WARN [IPC Server handler 4 on default port 35595 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-14T14:54:39,894 WARN [IPC Server handler 4 on default port 35595 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-14T14:54:39,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741867_1050 (size=17994) 2024-11-14T14:54:40,307 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/ec29e5b7b8574cebb9237b1ca52ef126 as hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/ec29e5b7b8574cebb9237b1ca52ef126 2024-11-14T14:54:40,314 INFO [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2979e3a8fec3327dfa64983b4f61c28/info of c2979e3a8fec3327dfa64983b4f61c28 into ec29e5b7b8574cebb9237b1ca52ef126(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T14:54:40,314 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2979e3a8fec3327dfa64983b4f61c28: 2024-11-14T14:54:40,314 INFO [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28., storeName=c2979e3a8fec3327dfa64983b4f61c28/info, priority=13, startTime=1731596079866; duration=0sec 2024-11-14T14:54:40,314 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-11-14T14:54:40,314 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:54:40,314 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/ec29e5b7b8574cebb9237b1ca52ef126 because midkey is the same as first or last row 2024-11-14T14:54:40,314 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-11-14T14:54:40,314 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:54:40,314 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/ec29e5b7b8574cebb9237b1ca52ef126 because midkey is the same as first or last row 2024-11-14T14:54:40,315 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-11-14T14:54:40,315 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:54:40,315 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/ec29e5b7b8574cebb9237b1ca52ef126 because midkey is the same as first or last row 2024-11-14T14:54:40,315 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:54:40,315 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2979e3a8fec3327dfa64983b4f61c28:info 2024-11-14T14:54:40,321 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@1bbdf1af[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44787, datanodeUuid=3961c212-814b-45b5-8059-b2afc9c559ae, infoPort=34017, infoSecurePort=0, ipcPort=41381, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577):Failed to transfer BP-1040518630-172.17.0.2-1731596058577:blk_1073741847_1030 to 127.0.0.1:33793 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:40,321 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@73f8b3a4[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44787, datanodeUuid=3961c212-814b-45b5-8059-b2afc9c559ae, infoPort=34017, infoSecurePort=0, ipcPort=41381, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577):Failed to transfer BP-1040518630-172.17.0.2-1731596058577:blk_1073741852_1035 to 127.0.0.1:33793 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:40,650 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40527 {}] regionserver.HRegion(8855): Flush requested on c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:40,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2979e3a8fec3327dfa64983b4f61c28 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-14T14:54:40,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/020a153a873b482c8ddb33239a192412 is 1079, key is tmprow/info:/1731596080853/Put/seqid=0 2024-11-14T14:54:40,862 WARN [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45899 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:40,862 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37336 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741868_1051] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data10]'}, localName='127.0.0.1:44787', datanodeUuid='3961c212-814b-45b5-8059-b2afc9c559ae', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741868_1051 to mirror 127.0.0.1:45899 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:40,862 WARN [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK], DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]) is bad. 2024-11-14T14:54:40,862 WARN [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741868_1051 2024-11-14T14:54:40,862 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37336 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741868_1051] {}] datanode.BlockReceiver(316): Block 1073741868 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-14T14:54:40,862 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37336 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741868_1051] {}] datanode.DataXceiver(331): 127.0.0.1:44787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37336 dst: /127.0.0.1:44787 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:40,863 WARN [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK] 2024-11-14T14:54:40,865 WARN [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40217 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:40,865 WARN [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK], DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]) is bad. 2024-11-14T14:54:40,865 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37352 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741869_1052] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data10]'}, localName='127.0.0.1:44787', datanodeUuid='3961c212-814b-45b5-8059-b2afc9c559ae', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741869_1052 to mirror 127.0.0.1:40217 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:40,865 WARN [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741869_1052 2024-11-14T14:54:40,865 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37352 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741869_1052] {}] datanode.BlockReceiver(316): Block 1073741869 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-14T14:54:40,866 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37352 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741869_1052] {}] datanode.DataXceiver(331): 127.0.0.1:44787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37352 dst: /127.0.0.1:44787 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:40,866 WARN [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK] 2024-11-14T14:54:40,867 WARN [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:40,867 WARN [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK], DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]) is bad. 2024-11-14T14:54:40,867 WARN [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741870_1053 2024-11-14T14:54:40,867 WARN [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK] 2024-11-14T14:54:40,869 WARN [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:40,869 WARN [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK], DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]) is bad. 2024-11-14T14:54:40,869 WARN [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741871_1054 2024-11-14T14:54:40,869 WARN [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK] 2024-11-14T14:54:40,870 WARN [IPC Server handler 1 on default port 35595 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-14T14:54:40,870 WARN [IPC Server handler 1 on default port 35595 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-14T14:54:40,870 WARN [IPC Server handler 1 on default port 35595 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-14T14:54:40,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741872_1055 (size=6027) 2024-11-14T14:54:41,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/020a153a873b482c8ddb33239a192412 2024-11-14T14:54:41,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/020a153a873b482c8ddb33239a192412 as hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/020a153a873b482c8ddb33239a192412 2024-11-14T14:54:41,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/020a153a873b482c8ddb33239a192412, entries=1, sequenceid=45, filesize=5.9 K 2024-11-14T14:54:41,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for c2979e3a8fec3327dfa64983b4f61c28 in 434ms, sequenceid=45, compaction requested=false 2024-11-14T14:54:41,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2979e3a8fec3327dfa64983b4f61c28: 2024-11-14T14:54:41,288 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K 2024-11-14T14:54:41,288 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:54:41,288 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/ec29e5b7b8574cebb9237b1ca52ef126 because midkey is the same as first or last row 2024-11-14T14:54:41,321 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@1bbdf1af[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44787, datanodeUuid=3961c212-814b-45b5-8059-b2afc9c559ae, infoPort=34017, infoSecurePort=0, ipcPort=41381, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577):Failed to transfer BP-1040518630-172.17.0.2-1731596058577:blk_1073741862_1045 to 127.0.0.1:45899 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:41,321 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@73f8b3a4[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44787, datanodeUuid=3961c212-814b-45b5-8059-b2afc9c559ae, infoPort=34017, infoSecurePort=0, ipcPort=41381, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577):Failed to transfer BP-1040518630-172.17.0.2-1731596058577:blk_1073741842_1025 to 127.0.0.1:33793 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:41,394 WARN [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK]] 2024-11-14T14:54:41,394 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:41,395 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog da1c6afcd1f9%2C40527%2C1731596059482:(num 1731596079378) roll requested 2024-11-14T14:54:41,395 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C40527%2C1731596059482.1731596081395 2024-11-14T14:54:41,398 WARN [Thread-961 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:41,399 WARN [Thread-961 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK], DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]) is bad. 2024-11-14T14:54:41,399 WARN [Thread-961 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741873_1056 2024-11-14T14:54:41,399 WARN [Thread-961 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK] 2024-11-14T14:54:41,401 WARN [Thread-961 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:41,401 WARN [Thread-961 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK], DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]) is bad. 2024-11-14T14:54:41,401 WARN [Thread-961 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741874_1057 2024-11-14T14:54:41,401 WARN [Thread-961 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK] 2024-11-14T14:54:41,404 WARN [Thread-961 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35719 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:41,404 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37372 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741875_1058] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data10]'}, localName='127.0.0.1:44787', datanodeUuid='3961c212-814b-45b5-8059-b2afc9c559ae', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741875_1058 to mirror 127.0.0.1:35719 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:41,404 WARN [Thread-961 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK], DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]) is bad. 2024-11-14T14:54:41,404 WARN [Thread-961 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741875_1058 2024-11-14T14:54:41,404 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37372 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741875_1058] {}] datanode.BlockReceiver(316): Block 1073741875 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-14T14:54:41,405 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37372 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741875_1058] {}] datanode.DataXceiver(331): 127.0.0.1:44787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37372 dst: /127.0.0.1:44787 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:41,405 WARN [Thread-961 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK] 2024-11-14T14:54:41,408 WARN [Thread-961 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45899 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:41,408 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37384 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741876_1059] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data10]'}, localName='127.0.0.1:44787', datanodeUuid='3961c212-814b-45b5-8059-b2afc9c559ae', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741876_1059 to mirror 127.0.0.1:45899 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:41,408 WARN [Thread-961 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK], DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]) is bad. 2024-11-14T14:54:41,408 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37384 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741876_1059] {}] datanode.BlockReceiver(316): Block 1073741876 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-14T14:54:41,408 WARN [Thread-961 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741876_1059 2024-11-14T14:54:41,408 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37384 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741876_1059] {}] datanode.DataXceiver(331): 127.0.0.1:44787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37384 dst: /127.0.0.1:44787 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:41,409 WARN [Thread-961 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK] 2024-11-14T14:54:41,409 WARN [IPC Server handler 1 on default port 35595 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-14T14:54:41,409 WARN [IPC Server handler 1 on default port 35595 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-14T14:54:41,409 WARN [IPC Server handler 1 on default port 35595 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-14T14:54:41,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:41,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:41,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:41,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:41,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:41,413 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596079378 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596081395 2024-11-14T14:54:41,414 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34017:34017)] 2024-11-14T14:54:41,414 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596060015 is not closed yet, will try archiving it next time 2024-11-14T14:54:41,414 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596079378 is not closed yet, will try archiving it next time 2024-11-14T14:54:41,414 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596075361 to hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/oldWALs/da1c6afcd1f9%2C40527%2C1731596059482.1731596075361 2024-11-14T14:54:41,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741857_1040 (size=13591) 2024-11-14T14:54:41,591 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:41,816 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596060015 is not closed yet, will try archiving it next time 2024-11-14T14:54:42,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40527 {}] regionserver.HRegion(8855): Flush requested on c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:42,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2979e3a8fec3327dfa64983b4f61c28 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-14T14:54:42,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/3e507a452eb14d4cb33701b80ddfb8cb is 1079, key is tmprow/info:/1731596082274/Put/seqid=0 2024-11-14T14:54:42,282 WARN [Thread-967 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:42,282 WARN [Thread-967 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK], DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]) is bad. 2024-11-14T14:54:42,282 WARN [Thread-967 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741878_1061 2024-11-14T14:54:42,283 WARN [Thread-967 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK] 2024-11-14T14:54:42,284 WARN [Thread-967 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1062 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:42,284 WARN [Thread-967 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741879_1062 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK], DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]) is bad. 2024-11-14T14:54:42,284 WARN [Thread-967 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741879_1062 2024-11-14T14:54:42,284 WARN [Thread-967 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK] 2024-11-14T14:54:42,286 WARN [Thread-967 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:42,286 WARN [Thread-967 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK], DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]) is bad. 2024-11-14T14:54:42,286 WARN [Thread-967 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741880_1063 2024-11-14T14:54:42,286 WARN [Thread-967 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK] 2024-11-14T14:54:42,288 WARN [Thread-967 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1064 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45899 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:42,288 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37414 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741881_1064] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data10]'}, localName='127.0.0.1:44787', datanodeUuid='3961c212-814b-45b5-8059-b2afc9c559ae', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741881_1064 to mirror 127.0.0.1:45899 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:42,288 WARN [Thread-967 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741881_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK], DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]) is bad. 2024-11-14T14:54:42,289 WARN [Thread-967 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741881_1064 2024-11-14T14:54:42,289 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37414 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741881_1064] {}] datanode.BlockReceiver(316): Block 1073741881 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-14T14:54:42,289 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37414 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741881_1064] {}] datanode.DataXceiver(331): 127.0.0.1:44787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37414 dst: /127.0.0.1:44787 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:42,289 WARN [Thread-967 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK] 2024-11-14T14:54:42,290 WARN [IPC Server handler 0 on default port 35595 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-14T14:54:42,290 WARN [IPC Server handler 0 on default port 35595 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-14T14:54:42,290 WARN [IPC Server handler 0 on default port 35595 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-14T14:54:42,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741882_1065 (size=6027) 2024-11-14T14:54:42,651 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:42,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/3e507a452eb14d4cb33701b80ddfb8cb 2024-11-14T14:54:42,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/3e507a452eb14d4cb33701b80ddfb8cb as hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/3e507a452eb14d4cb33701b80ddfb8cb 2024-11-14T14:54:42,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/3e507a452eb14d4cb33701b80ddfb8cb, entries=1, sequenceid=55, filesize=5.9 K 2024-11-14T14:54:42,707 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for c2979e3a8fec3327dfa64983b4f61c28 in 431ms, sequenceid=55, compaction requested=true 2024-11-14T14:54:42,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2979e3a8fec3327dfa64983b4f61c28: 2024-11-14T14:54:42,707 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K 2024-11-14T14:54:42,707 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:54:42,707 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/ec29e5b7b8574cebb9237b1ca52ef126 because midkey is the same as first or last row 2024-11-14T14:54:42,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2979e3a8fec3327dfa64983b4f61c28:info, priority=-2147483648, current under compaction store size is 1 2024-11-14T14:54:42,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:54:42,707 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T14:54:42,708 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T14:54:42,709 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.HStore(1541): c2979e3a8fec3327dfa64983b4f61c28/info is initiating minor compaction (all files) 2024-11-14T14:54:42,709 INFO [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2979e3a8fec3327dfa64983b4f61c28/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28. 2024-11-14T14:54:42,709 INFO [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/ec29e5b7b8574cebb9237b1ca52ef126, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/020a153a873b482c8ddb33239a192412, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/3e507a452eb14d4cb33701b80ddfb8cb] into tmpdir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp, totalSize=29.3 K 2024-11-14T14:54:42,709 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec29e5b7b8574cebb9237b1ca52ef126, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731596073371 2024-11-14T14:54:42,710 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] compactions.Compactor(225): Compacting 020a153a873b482c8ddb33239a192412, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731596080853 2024-11-14T14:54:42,710 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e507a452eb14d4cb33701b80ddfb8cb, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731596082274 2024-11-14T14:54:42,725 INFO [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2979e3a8fec3327dfa64983b4f61c28#info#compaction#24 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T14:54:42,725 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/832a1616aee94a24ba44330be1220be3 is 1080, key is row0002/info:/1731596073371/Put/seqid=0 2024-11-14T14:54:42,728 WARN [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:45899 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:42,728 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37442 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741883_1066] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data10]'}, localName='127.0.0.1:44787', datanodeUuid='3961c212-814b-45b5-8059-b2afc9c559ae', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741883_1066 to mirror 127.0.0.1:45899 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:42,728 WARN [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK], DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]) is bad. 2024-11-14T14:54:42,728 WARN [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741883_1066 2024-11-14T14:54:42,728 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37442 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741883_1066] {}] datanode.BlockReceiver(316): Block 1073741883 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-14T14:54:42,728 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37442 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741883_1066] {}] datanode.DataXceiver(331): 127.0.0.1:44787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37442 dst: /127.0.0.1:44787 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:42,729 WARN [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK] 2024-11-14T14:54:42,730 WARN [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741884_1067 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:42,730 WARN [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741884_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK], DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]) is bad. 2024-11-14T14:54:42,730 WARN [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741884_1067 2024-11-14T14:54:42,730 WARN [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK] 2024-11-14T14:54:42,731 WARN [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1068 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:42,731 WARN [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741885_1068 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK], DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]) is bad. 2024-11-14T14:54:42,731 WARN [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741885_1068 2024-11-14T14:54:42,732 WARN [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35719,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK] 2024-11-14T14:54:42,734 WARN [Thread-972 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741886_1069 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33793 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:42,734 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37456 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741886_1069] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data10]'}, localName='127.0.0.1:44787', datanodeUuid='3961c212-814b-45b5-8059-b2afc9c559ae', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741886_1069 to mirror 127.0.0.1:33793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:42,734 WARN [Thread-972 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741886_1069 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK], DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]) is bad. 2024-11-14T14:54:42,734 WARN [Thread-972 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741886_1069 2024-11-14T14:54:42,734 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37456 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741886_1069] {}] datanode.BlockReceiver(316): Block 1073741886 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-14T14:54:42,734 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:37456 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741886_1069] {}] datanode.DataXceiver(331): 127.0.0.1:44787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37456 dst: /127.0.0.1:44787 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:42,734 WARN [Thread-972 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK] 2024-11-14T14:54:42,735 WARN [IPC Server handler 4 on default port 35595 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-14T14:54:42,735 WARN [IPC Server handler 4 on default port 35595 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-14T14:54:42,735 WARN [IPC Server handler 4 on default port 35595 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-14T14:54:42,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741887_1070 (size=18097) 2024-11-14T14:54:43,147 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/832a1616aee94a24ba44330be1220be3 as hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/832a1616aee94a24ba44330be1220be3 2024-11-14T14:54:43,154 INFO [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2979e3a8fec3327dfa64983b4f61c28/info of c2979e3a8fec3327dfa64983b4f61c28 into 832a1616aee94a24ba44330be1220be3(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T14:54:43,154 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2979e3a8fec3327dfa64983b4f61c28: 2024-11-14T14:54:43,154 INFO [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28., storeName=c2979e3a8fec3327dfa64983b4f61c28/info, priority=13, startTime=1731596082707; duration=0sec 2024-11-14T14:54:43,154 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K 2024-11-14T14:54:43,154 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:54:43,154 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/832a1616aee94a24ba44330be1220be3 because midkey is the same as first or last row 2024-11-14T14:54:43,154 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K 2024-11-14T14:54:43,154 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:54:43,154 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/832a1616aee94a24ba44330be1220be3 because midkey is the same as first or last row 2024-11-14T14:54:43,154 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K 2024-11-14T14:54:43,154 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:54:43,154 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/832a1616aee94a24ba44330be1220be3 because midkey is the same as first or last row 2024-11-14T14:54:43,154 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:54:43,154 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2979e3a8fec3327dfa64983b4f61c28:info 2024-11-14T14:54:43,322 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@73f8b3a4[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44787, datanodeUuid=3961c212-814b-45b5-8059-b2afc9c559ae, infoPort=34017, infoSecurePort=0, ipcPort=41381, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577):Failed to transfer BP-1040518630-172.17.0.2-1731596058577:blk_1073741867_1050 to 127.0.0.1:40217 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:43,322 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@1bbdf1af[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44787, datanodeUuid=3961c212-814b-45b5-8059-b2afc9c559ae, infoPort=34017, infoSecurePort=0, ipcPort=41381, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577):Failed to transfer BP-1040518630-172.17.0.2-1731596058577:blk_1073741872_1055 to 127.0.0.1:40217 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:43,414 WARN [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas. 2024-11-14T14:54:43,414 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:43,500 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:54:43,503 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:54:43,503 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:54:43,503 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:54:43,503 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:54:43,504 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2aaad1eb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:54:43,504 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@719d00c0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:54:43,592 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:43,620 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5536ce0c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/java.io.tmpdir/jetty-localhost-39687-hadoop-hdfs-3_4_1-tests_jar-_-any-14891437577935568557/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:54:43,620 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4c03d0a2{HTTP/1.1, (http/1.1)}{localhost:39687} 2024-11-14T14:54:43,620 INFO [Time-limited test {}] server.Server(415): Started @129242ms 2024-11-14T14:54:43,621 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:54:43,729 WARN [Thread-992 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:54:43,737 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9d6a32729d735060 with lease ID 0x2ad170a756447207: from storage DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015 node DatanodeRegistration(127.0.0.1:38639, datanodeUuid=ed0576a8-2ece-4054-b155-ea617541c969, infoPort=33533, infoSecurePort=0, ipcPort=33477, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T14:54:43,738 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9d6a32729d735060 with lease ID 0x2ad170a756447207: from storage DS-0771e6af-5153-420f-a8b9-069a2555649a node DatanodeRegistration(127.0.0.1:38639, datanodeUuid=ed0576a8-2ece-4054-b155-ea617541c969, infoPort=33533, infoSecurePort=0, ipcPort=33477, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:54:44,321 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@1bbdf1af[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44787, datanodeUuid=3961c212-814b-45b5-8059-b2afc9c559ae, infoPort=34017, infoSecurePort=0, ipcPort=41381, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577):Failed to transfer BP-1040518630-172.17.0.2-1731596058577:blk_1073741857_1040 to 127.0.0.1:33793 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:44,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38639 is added to blk_1073741882_1065 (size=6027) 2024-11-14T14:54:44,651 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:45,415 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:45,592 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:46,321 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@73f8b3a4[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44787, datanodeUuid=3961c212-814b-45b5-8059-b2afc9c559ae, infoPort=34017, infoSecurePort=0, ipcPort=41381, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577):Failed to transfer BP-1040518630-172.17.0.2-1731596058577:blk_1073741887_1070 to 127.0.0.1:40217 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:46,651 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:47,415 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:47,592 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:48,652 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:49,383 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T14:54:49,416 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:49,593 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:49,733 ERROR [FSHLog-0-hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData-prefix:da1c6afcd1f9,40173,1731596059405 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:49,733 WARN [FSHLog-0-hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData-prefix:da1c6afcd1f9,40173,1731596059405 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:49,733 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog da1c6afcd1f9%2C40173%2C1731596059405:(num 1731596059635) roll requested 2024-11-14T14:54:49,734 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C40173%2C1731596059405.1731596089733 2024-11-14T14:54:49,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:49,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:49,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:49,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:49,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:49,741 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/WALs/da1c6afcd1f9,40173,1731596059405/da1c6afcd1f9%2C40173%2C1731596059405.1731596059635 with entries=54, filesize=26.68 KB; new WAL /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/WALs/da1c6afcd1f9,40173,1731596059405/da1c6afcd1f9%2C40173%2C1731596059405.1731596089733 2024-11-14T14:54:49,741 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:49,741 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:49,741 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/WALs/da1c6afcd1f9,40173,1731596059405/da1c6afcd1f9%2C40173%2C1731596059405.1731596059635 2024-11-14T14:54:49,742 WARN [IPC Server handler 1 on default port 35595 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/WALs/da1c6afcd1f9,40173,1731596059405/da1c6afcd1f9%2C40173%2C1731596059405.1731596059635 has not been closed. Lease recovery is in progress. RecoveryId = 1072 for block blk_1073741830_1006 2024-11-14T14:54:49,742 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34017:34017),(127.0.0.1/127.0.0.1:33533:33533)] 2024-11-14T14:54:49,742 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/WALs/da1c6afcd1f9,40173,1731596059405/da1c6afcd1f9%2C40173%2C1731596059405.1731596059635 after 1ms 2024-11-14T14:54:49,742 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/WALs/da1c6afcd1f9,40173,1731596059405/da1c6afcd1f9%2C40173%2C1731596059405.1731596059635 is not closed yet, will try archiving it next time 2024-11-14T14:54:50,652 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:51,416 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:52,652 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:53,416 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:53,743 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/WALs/da1c6afcd1f9,40173,1731596059405/da1c6afcd1f9%2C40173%2C1731596059405.1731596059635 after 4002ms 2024-11-14T14:54:53,753 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1aab9cfd {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1040518630-172.17.0.2-1731596058577:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:33793,null,null]) java.net.ConnectException: Call From da1c6afcd1f9/172.17.0.2 to localhost:43867 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-14T14:54:53,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38639 is added to blk_1073741833_1020 (size=455) 2024-11-14T14:54:54,384 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596060015 to hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/oldWALs/da1c6afcd1f9%2C40527%2C1731596059482.1731596060015 2024-11-14T14:54:54,386 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596079378 to hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/oldWALs/da1c6afcd1f9%2C40527%2C1731596059482.1731596079378 2024-11-14T14:54:54,653 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:55,417 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:55,733 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@2b7bc2d0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:38639, datanodeUuid=ed0576a8-2ece-4054-b155-ea617541c969, infoPort=33533, infoSecurePort=0, ipcPort=33477, storageInfo=lv=-57;cid=testClusterID;nsid=200699414;c=1731596058577):Failed to transfer BP-1040518630-172.17.0.2-1731596058577:blk_1073741833_1020 to 127.0.0.1:40217 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:56,653 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,049 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C40527%2C1731596059482.1731596097049 2024-11-14T14:54:57,052 WARN [Thread-1023 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741889_1073 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,052 WARN [Thread-1023 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741889_1073 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK], DatanodeInfoWithStorage[127.0.0.1:38639,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]) is bad. 2024-11-14T14:54:57,052 WARN [Thread-1023 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741889_1073 2024-11-14T14:54:57,054 WARN [Thread-1023 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK] 2024-11-14T14:54:57,055 WARN [Thread-1023 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741890_1074 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,055 WARN [Thread-1023 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741890_1074 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK], DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]) is bad. 2024-11-14T14:54:57,055 WARN [Thread-1023 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741890_1074 2024-11-14T14:54:57,056 WARN [Thread-1023 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK] 2024-11-14T14:54:57,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,061 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596081395 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596097049 2024-11-14T14:54:57,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741877_1060 (size=12911) 2024-11-14T14:54:57,067 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33533:33533),(127.0.0.1/127.0.0.1:34017:34017)] 2024-11-14T14:54:57,067 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596081395 is not closed yet, will try archiving it next time 2024-11-14T14:54:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40527 {}] regionserver.HRegion(8855): Flush requested on c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:57,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2979e3a8fec3327dfa64983b4f61c28 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-14T14:54:57,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/6e7391c6cbac443a955ddcb598c5d5c9 is 1080, key is row0013/info:/1731596097068/Put/seqid=0 2024-11-14T14:54:57,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38639 is added to blk_1073741892_1076 (size=8190) 2024-11-14T14:54:57,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741892_1076 (size=8190) 2024-11-14T14:54:57,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/6e7391c6cbac443a955ddcb598c5d5c9 2024-11-14T14:54:57,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/6e7391c6cbac443a955ddcb598c5d5c9 as hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/6e7391c6cbac443a955ddcb598c5d5c9 2024-11-14T14:54:57,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/6e7391c6cbac443a955ddcb598c5d5c9, entries=3, sequenceid=66, filesize=8.0 K 2024-11-14T14:54:57,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9683 for c2979e3a8fec3327dfa64983b4f61c28 in 29ms, sequenceid=66, compaction requested=false 2024-11-14T14:54:57,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2979e3a8fec3327dfa64983b4f61c28: 2024-11-14T14:54:57,102 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K 2024-11-14T14:54:57,102 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:54:57,102 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/832a1616aee94a24ba44330be1220be3 because midkey is the same as first or last row 2024-11-14T14:54:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40527 {}] regionserver.HRegion(8855): Flush requested on c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:57,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c2979e3a8fec3327dfa64983b4f61c28 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB 2024-11-14T14:54:57,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/331fe2f484e94cd48477f0840b5bba5f is 1080, key is row0015/info:/1731596097073/Put/seqid=0 2024-11-14T14:54:57,300 WARN [Thread-1037 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741893_1077 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40217 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,300 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54874 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741893_1077] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data10]'}, localName='127.0.0.1:44787', datanodeUuid='3961c212-814b-45b5-8059-b2afc9c559ae', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741893_1077 to mirror 127.0.0.1:40217 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:57,300 WARN [Thread-1037 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741893_1077 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK], DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]) is bad. 2024-11-14T14:54:57,300 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54874 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741893_1077] {}] datanode.BlockReceiver(316): Block 1073741893 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-14T14:54:57,300 WARN [Thread-1037 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741893_1077 2024-11-14T14:54:57,301 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54874 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741893_1077] {}] datanode.DataXceiver(331): 127.0.0.1:44787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54874 dst: /127.0.0.1:44787 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:57,301 WARN [Thread-1037 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK] 2024-11-14T14:54:57,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38639 is added to blk_1073741894_1078 (size=14660) 2024-11-14T14:54:57,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741894_1078 (size=14660) 2024-11-14T14:54:57,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/331fe2f484e94cd48477f0840b5bba5f 2024-11-14T14:54:57,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/331fe2f484e94cd48477f0840b5bba5f as hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/331fe2f484e94cd48477f0840b5bba5f 2024-11-14T14:54:57,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/331fe2f484e94cd48477f0840b5bba5f, entries=9, sequenceid=79, filesize=14.3 K 2024-11-14T14:54:57,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10758, heapSize ~11.48 KB/11760, currentSize=0 B/0 for c2979e3a8fec3327dfa64983b4f61c28 in 27ms, sequenceid=79, compaction requested=true 2024-11-14T14:54:57,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c2979e3a8fec3327dfa64983b4f61c28: 2024-11-14T14:54:57,319 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.0 K, sizeToCheck=16.0 K 2024-11-14T14:54:57,319 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:54:57,319 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/832a1616aee94a24ba44330be1220be3 because midkey is the same as first or last row 2024-11-14T14:54:57,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c2979e3a8fec3327dfa64983b4f61c28:info, priority=-2147483648, current under compaction store size is 1 2024-11-14T14:54:57,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:54:57,320 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T14:54:57,320 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T14:54:57,321 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.HStore(1541): c2979e3a8fec3327dfa64983b4f61c28/info is initiating minor compaction (all files) 2024-11-14T14:54:57,321 INFO [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c2979e3a8fec3327dfa64983b4f61c28/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28. 2024-11-14T14:54:57,321 INFO [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/832a1616aee94a24ba44330be1220be3, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/6e7391c6cbac443a955ddcb598c5d5c9, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/331fe2f484e94cd48477f0840b5bba5f] into tmpdir=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp, totalSize=40.0 K 2024-11-14T14:54:57,321 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] compactions.Compactor(225): Compacting 832a1616aee94a24ba44330be1220be3, keycount=12, bloomtype=ROW, size=17.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731596073371 2024-11-14T14:54:57,322 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e7391c6cbac443a955ddcb598c5d5c9, keycount=3, bloomtype=ROW, size=8.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731596083289 2024-11-14T14:54:57,322 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] compactions.Compactor(225): Compacting 331fe2f484e94cd48477f0840b5bba5f, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731596097073 2024-11-14T14:54:57,334 INFO [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c2979e3a8fec3327dfa64983b4f61c28#info#compaction#27 average throughput is 11.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T14:54:57,335 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/ce499f5acfd24ea59bb96e7305866775 is 1080, key is row0002/info:/1731596073371/Put/seqid=0 2024-11-14T14:54:57,336 WARN [Thread-1045 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741895_1079 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,336 WARN [Thread-1045 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741895_1079 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK], DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]) is bad. 2024-11-14T14:54:57,336 WARN [Thread-1045 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741895_1079 2024-11-14T14:54:57,337 WARN [Thread-1045 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK] 2024-11-14T14:54:57,338 WARN [Thread-1045 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741896_1080 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,338 WARN [Thread-1045 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741896_1080 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK], DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]) is bad. 2024-11-14T14:54:57,338 WARN [Thread-1045 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741896_1080 2024-11-14T14:54:57,339 WARN [Thread-1045 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK] 2024-11-14T14:54:57,341 WARN [Thread-1045 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741897_1081 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33793 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,341 WARN [Thread-1045 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741897_1081 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38639,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK], DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]) is bad. 2024-11-14T14:54:57,341 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:44138 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741897_1081] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data4]'}, localName='127.0.0.1:38639', datanodeUuid='ed0576a8-2ece-4054-b155-ea617541c969', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741897_1081 to mirror 127.0.0.1:33793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:57,341 WARN [Thread-1045 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741897_1081 2024-11-14T14:54:57,341 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:44138 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741897_1081] {}] datanode.BlockReceiver(316): Block 1073741897 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-14T14:54:57,341 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:44138 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741897_1081] {}] datanode.DataXceiver(331): 127.0.0.1:38639:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44138 dst: /127.0.0.1:38639 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:57,342 WARN [Thread-1045 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK] 2024-11-14T14:54:57,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741898_1082 (size=28989) 2024-11-14T14:54:57,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38639 is added to blk_1073741898_1082 (size=28989) 2024-11-14T14:54:57,355 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/.tmp/info/ce499f5acfd24ea59bb96e7305866775 as hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/ce499f5acfd24ea59bb96e7305866775 2024-11-14T14:54:57,362 INFO [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c2979e3a8fec3327dfa64983b4f61c28/info of c2979e3a8fec3327dfa64983b4f61c28 into ce499f5acfd24ea59bb96e7305866775(size=28.3 K), total size for store is 28.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T14:54:57,362 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c2979e3a8fec3327dfa64983b4f61c28: 2024-11-14T14:54:57,362 INFO [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28., storeName=c2979e3a8fec3327dfa64983b4f61c28/info, priority=13, startTime=1731596097319; duration=0sec 2024-11-14T14:54:57,362 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K 2024-11-14T14:54:57,362 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:54:57,362 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/ce499f5acfd24ea59bb96e7305866775 because midkey is the same as first or last row 2024-11-14T14:54:57,362 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K 2024-11-14T14:54:57,362 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:54:57,362 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/ce499f5acfd24ea59bb96e7305866775 because midkey is the same as first or last row 2024-11-14T14:54:57,362 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K 2024-11-14T14:54:57,362 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:54:57,362 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/ce499f5acfd24ea59bb96e7305866775 because midkey is the same as first or last row 2024-11-14T14:54:57,362 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:54:57,362 DEBUG [RS:0;da1c6afcd1f9:40527-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c2979e3a8fec3327dfa64983b4f61c28:info 2024-11-14T14:54:57,417 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled. 2024-11-14T14:54:57,417 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,463 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.1731596081395 to hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/oldWALs/da1c6afcd1f9%2C40527%2C1731596059482.1731596081395 2024-11-14T14:54:57,492 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T14:54:57,493 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:54:57,493 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:54:57,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:54:57,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:54:57,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T14:54:57,493 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T14:54:57,493 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=983930394, stopped=false 2024-11-14T14:54:57,493 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=da1c6afcd1f9,40173,1731596059405 2024-11-14T14:54:57,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:54:57,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:54:57,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34075-0x101a74678630002, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:54:57,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:57,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:57,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34075-0x101a74678630002, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:54:57,495 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:54:57,496 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:54:57,496 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:54:57,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:54:57,496 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'da1c6afcd1f9,40527,1731596059482' ***** 2024-11-14T14:54:57,496 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T14:54:57,496 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'da1c6afcd1f9,34075,1731596060544' ***** 2024-11-14T14:54:57,496 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T14:54:57,496 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:54:57,496 INFO [RS:1;da1c6afcd1f9:34075 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T14:54:57,496 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T14:54:57,497 INFO [RS:1;da1c6afcd1f9:34075 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T14:54:57,497 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T14:54:57,497 INFO [RS:0;da1c6afcd1f9:40527 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T14:54:57,497 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T14:54:57,497 INFO [RS:1;da1c6afcd1f9:34075 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T14:54:57,497 INFO [RS:0;da1c6afcd1f9:40527 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T14:54:57,497 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:54:57,497 INFO [RS:1;da1c6afcd1f9:34075 {}] regionserver.HRegionServer(959): stopping server da1c6afcd1f9,34075,1731596060544 2024-11-14T14:54:57,497 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.HRegionServer(3091): Received CLOSE for c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:57,497 INFO [RS:1;da1c6afcd1f9:34075 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:54:57,497 INFO [RS:1;da1c6afcd1f9:34075 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;da1c6afcd1f9:34075. 2024-11-14T14:54:57,497 DEBUG [RS:1;da1c6afcd1f9:34075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:54:57,497 DEBUG [RS:1;da1c6afcd1f9:34075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:54:57,497 INFO [RS:1;da1c6afcd1f9:34075 {}] regionserver.HRegionServer(976): stopping server da1c6afcd1f9,34075,1731596060544; all regions closed. 2024-11-14T14:54:57,497 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34075-0x101a74678630002, quorum=127.0.0.1:60317, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:54:57,497 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.HRegionServer(959): stopping server da1c6afcd1f9,40527,1731596059482 2024-11-14T14:54:57,497 INFO [RS:0;da1c6afcd1f9:40527 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:54:57,497 INFO [RS:0;da1c6afcd1f9:40527 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;da1c6afcd1f9:40527. 2024-11-14T14:54:57,498 DEBUG [RS:0;da1c6afcd1f9:40527 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:54:57,498 DEBUG [RS:0;da1c6afcd1f9:40527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:54:57,498 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T14:54:57,498 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T14:54:57,498 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T14:54:57,498 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T14:54:57,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,498 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c2979e3a8fec3327dfa64983b4f61c28, disabling compactions & flushes 2024-11-14T14:54:57,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,498 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-14T14:54:57,498 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28. 2024-11-14T14:54:57,498 DEBUG [RS:0;da1c6afcd1f9:40527 {}] regionserver.HRegionServer(1325): Online Regions={c2979e3a8fec3327dfa64983b4f61c28=TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28., 1588230740=hbase:meta,,1.1588230740} 2024-11-14T14:54:57,498 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28. 2024-11-14T14:54:57,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,498 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28. after waiting 0 ms 2024-11-14T14:54:57,498 DEBUG [RS:0;da1c6afcd1f9:40527 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c2979e3a8fec3327dfa64983b4f61c28 2024-11-14T14:54:57,498 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28. 2024-11-14T14:54:57,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,498 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:54:57,499 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:54:57,499 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:54:57,499 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:54:57,499 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:54:57,499 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB 2024-11-14T14:54:57,499 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/14bf962eb0c8455b8bf68c0fbbc1a5ec, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/63a940bb12f446fb9fcf3d733d803b1a, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/ec29e5b7b8574cebb9237b1ca52ef126, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/1d1f7cb7b6144f51ac1d9933aef40d45, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/020a153a873b482c8ddb33239a192412, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/832a1616aee94a24ba44330be1220be3, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/3e507a452eb14d4cb33701b80ddfb8cb, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/6e7391c6cbac443a955ddcb598c5d5c9, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/331fe2f484e94cd48477f0840b5bba5f] to archive 2024-11-14T14:54:57,499 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,499 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,499 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 2024-11-14T14:54:57,499 ERROR [FSHLog-0-hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7-prefix:da1c6afcd1f9,40527,1731596059482.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,499 WARN [FSHLog-0-hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7-prefix:da1c6afcd1f9,40527,1731596059482.meta {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,500 WARN [IPC Server handler 0 on default port 35595 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 has not been closed. Lease recovery is in progress. RecoveryId = 1083 for block blk_1073741837_1013 2024-11-14T14:54:57,500 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog da1c6afcd1f9%2C40527%2C1731596059482.meta:.meta(num 1731596060396) roll requested 2024-11-14T14:54:57,500 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 after 1ms 2024-11-14T14:54:57,500 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596097500.meta 2024-11-14T14:54:57,500 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T14:54:57,502 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/14bf962eb0c8455b8bf68c0fbbc1a5ec to hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/14bf962eb0c8455b8bf68c0fbbc1a5ec 2024-11-14T14:54:57,502 WARN [Thread-1052 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741899_1084 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,503 WARN [Thread-1052 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741899_1084 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK], DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]) is bad. 2024-11-14T14:54:57,503 WARN [Thread-1052 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741899_1084 2024-11-14T14:54:57,503 WARN [Thread-1052 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK] 2024-11-14T14:54:57,504 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/63a940bb12f446fb9fcf3d733d803b1a to hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/63a940bb12f446fb9fcf3d733d803b1a 2024-11-14T14:54:57,505 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/ec29e5b7b8574cebb9237b1ca52ef126 to hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/ec29e5b7b8574cebb9237b1ca52ef126 2024-11-14T14:54:57,505 WARN [Thread-1052 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741900_1085 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33793 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,505 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:44150 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741900_1085] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data4]'}, localName='127.0.0.1:38639', datanodeUuid='ed0576a8-2ece-4054-b155-ea617541c969', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741900_1085 to mirror 127.0.0.1:33793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:57,505 WARN [Thread-1052 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741900_1085 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38639,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK], DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]) is bad. 2024-11-14T14:54:57,505 WARN [Thread-1052 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741900_1085 2024-11-14T14:54:57,505 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:44150 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741900_1085] {}] datanode.BlockReceiver(316): Block 1073741900 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-14T14:54:57,506 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:44150 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741900_1085] {}] datanode.DataXceiver(331): 127.0.0.1:38639:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44150 dst: /127.0.0.1:38639 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:57,506 WARN [Thread-1052 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK] 2024-11-14T14:54:57,506 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/1d1f7cb7b6144f51ac1d9933aef40d45 to hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/1d1f7cb7b6144f51ac1d9933aef40d45 2024-11-14T14:54:57,508 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/020a153a873b482c8ddb33239a192412 to hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/020a153a873b482c8ddb33239a192412 2024-11-14T14:54:57,509 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/832a1616aee94a24ba44330be1220be3 to hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/832a1616aee94a24ba44330be1220be3 2024-11-14T14:54:57,510 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/3e507a452eb14d4cb33701b80ddfb8cb to hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/3e507a452eb14d4cb33701b80ddfb8cb 2024-11-14T14:54:57,512 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/6e7391c6cbac443a955ddcb598c5d5c9 to hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/6e7391c6cbac443a955ddcb598c5d5c9 2024-11-14T14:54:57,513 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/331fe2f484e94cd48477f0840b5bba5f to hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/info/331fe2f484e94cd48477f0840b5bba5f 2024-11-14T14:54:57,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,513 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=da1c6afcd1f9:40173 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 16 more 2024-11-14T14:54:57,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,514 WARN [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [14bf962eb0c8455b8bf68c0fbbc1a5ec=10347, 63a940bb12f446fb9fcf3d733d803b1a=12506, ec29e5b7b8574cebb9237b1ca52ef126=17994, 1d1f7cb7b6144f51ac1d9933aef40d45=6027, 020a153a873b482c8ddb33239a192412=6027, 832a1616aee94a24ba44330be1220be3=18097, 3e507a452eb14d4cb33701b80ddfb8cb=6027, 6e7391c6cbac443a955ddcb598c5d5c9=8190, 331fe2f484e94cd48477f0840b5bba5f=14660] 2024-11-14T14:54:57,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,514 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596097500.meta 2024-11-14T14:54:57,515 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,515 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,515 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta 2024-11-14T14:54:57,515 WARN [IPC Server handler 3 on default port 35595 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta has not been closed. Lease recovery is in progress. RecoveryId = 1087 for block blk_1073741834_1010 2024-11-14T14:54:57,515 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34017:34017),(127.0.0.1/127.0.0.1:33533:33533)] 2024-11-14T14:54:57,515 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta is not closed yet, will try archiving it next time 2024-11-14T14:54:57,515 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta after 0ms 2024-11-14T14:54:57,518 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/default/TestLogRolling-testLogRollOnDatanodeDeath/c2979e3a8fec3327dfa64983b4f61c28/recovered.edits/83.seqid, newMaxSeqId=83, maxSeqId=1 2024-11-14T14:54:57,519 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28. 2024-11-14T14:54:57,519 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c2979e3a8fec3327dfa64983b4f61c28: Waiting for close lock at 1731596097498Running coprocessor pre-close hooks at 1731596097498Disabling compacts and flushes for region at 1731596097498Disabling writes for close at 1731596097498Writing region close event to WAL at 1731596097514 (+16 ms)Running coprocessor post-close hooks at 1731596097519 (+5 ms)Closed at 1731596097519 2024-11-14T14:54:57,519 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28. 2024-11-14T14:54:57,531 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740/.tmp/info/4d106002523644dc9bbed855557f2416 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1731596060711.c2979e3a8fec3327dfa64983b4f61c28./info:regioninfo/1731596061115/Put/seqid=0 2024-11-14T14:54:57,533 WARN [Thread-1060 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741902_1088 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,533 WARN [Thread-1060 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741902_1088 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK], DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]) is bad. 2024-11-14T14:54:57,534 WARN [Thread-1060 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741902_1088 2024-11-14T14:54:57,534 WARN [Thread-1060 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK] 2024-11-14T14:54:57,535 WARN [Thread-1060 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741903_1089 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,535 WARN [Thread-1060 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741903_1089 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK], DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]) is bad. 2024-11-14T14:54:57,535 WARN [Thread-1060 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741903_1089 2024-11-14T14:54:57,536 WARN [Thread-1060 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK] 2024-11-14T14:54:57,537 WARN [Thread-1060 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741904_1090 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,537 WARN [Thread-1060 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741904_1090 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK], DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]) is bad. 2024-11-14T14:54:57,537 WARN [Thread-1060 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741904_1090 2024-11-14T14:54:57,538 WARN [Thread-1060 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK] 2024-11-14T14:54:57,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38639 is added to blk_1073741905_1091 (size=7089) 2024-11-14T14:54:57,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741905_1091 (size=7089) 2024-11-14T14:54:57,543 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740/.tmp/info/4d106002523644dc9bbed855557f2416 2024-11-14T14:54:57,569 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740/.tmp/ns/428a70b29de1472982e0999e2779d4f9 is 43, key is default/ns:d/1731596060446/Put/seqid=0 2024-11-14T14:54:57,572 WARN [Thread-1066 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741906_1092 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40217 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,572 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:44184 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741906_1092] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data4]'}, localName='127.0.0.1:38639', datanodeUuid='ed0576a8-2ece-4054-b155-ea617541c969', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741906_1092 to mirror 127.0.0.1:40217 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:57,572 WARN [Thread-1066 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741906_1092 in pipeline [DatanodeInfoWithStorage[127.0.0.1:38639,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK], DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK]) is bad. 2024-11-14T14:54:57,572 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:44184 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741906_1092] {}] datanode.BlockReceiver(316): Block 1073741906 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-14T14:54:57,572 WARN [Thread-1066 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741906_1092 2024-11-14T14:54:57,572 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:44184 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741906_1092] {}] datanode.DataXceiver(331): 127.0.0.1:38639:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44184 dst: /127.0.0.1:38639 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:57,573 WARN [Thread-1066 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40217,DS-ab0c677b-1249-4dbd-a7c5-65698bee7202,DISK] 2024-11-14T14:54:57,574 WARN [Thread-1066 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741907_1093 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,574 WARN [Thread-1066 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741907_1093 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK], DatanodeInfoWithStorage[127.0.0.1:38639,DS-ebe3a6cd-5472-4be8-b92a-1b6335eba015,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK]) is bad. 2024-11-14T14:54:57,574 WARN [Thread-1066 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741907_1093 2024-11-14T14:54:57,574 WARN [Thread-1066 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:45899,DS-dd0a4651-76de-4a3c-8a78-5e22a04e6c02,DISK] 2024-11-14T14:54:57,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38639 is added to blk_1073741908_1094 (size=5153) 2024-11-14T14:54:57,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741908_1094 (size=5153) 2024-11-14T14:54:57,579 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740/.tmp/ns/428a70b29de1472982e0999e2779d4f9 2024-11-14T14:54:57,606 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740/.tmp/table/3bd71f1b31374a54baa414b8add50fcb is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1731596061126/Put/seqid=0 2024-11-14T14:54:57,609 WARN [Thread-1073 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741909_1095 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:33793 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:54:57,609 WARN [Thread-1073 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1040518630-172.17.0.2-1731596058577:blk_1073741909_1095 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44787,DS-971a76db-9204-4270-ba46-5b1009f5de99,DISK], DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK]) is bad. 2024-11-14T14:54:57,609 WARN [Thread-1073 {}] hdfs.DataStreamer(1850): Abandoning BP-1040518630-172.17.0.2-1731596058577:blk_1073741909_1095 2024-11-14T14:54:57,609 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54926 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741909_1095] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data9, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data10]'}, localName='127.0.0.1:44787', datanodeUuid='3961c212-814b-45b5-8059-b2afc9c559ae', xmitsInProgress=0}:Exception transferring block BP-1040518630-172.17.0.2-1731596058577:blk_1073741909_1095 to mirror 127.0.0.1:33793 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:57,609 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54926 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741909_1095] {}] datanode.BlockReceiver(316): Block 1073741909 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-14T14:54:57,610 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_797090476_22 at /127.0.0.1:54926 [Receiving block BP-1040518630-172.17.0.2-1731596058577:blk_1073741909_1095] {}] datanode.DataXceiver(331): 127.0.0.1:44787:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54926 dst: /127.0.0.1:44787 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:54:57,610 WARN [Thread-1073 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:33793,DS-a53a5f2f-c38d-4681-804e-6e92bb8ffbea,DISK] 2024-11-14T14:54:57,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38639 is added to blk_1073741910_1096 (size=5424) 2024-11-14T14:54:57,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741910_1096 (size=5424) 2024-11-14T14:54:57,618 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740/.tmp/table/3bd71f1b31374a54baa414b8add50fcb 2024-11-14T14:54:57,624 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740/.tmp/info/4d106002523644dc9bbed855557f2416 as hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740/info/4d106002523644dc9bbed855557f2416 2024-11-14T14:54:57,630 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740/info/4d106002523644dc9bbed855557f2416, entries=10, sequenceid=11, filesize=6.9 K 2024-11-14T14:54:57,631 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740/.tmp/ns/428a70b29de1472982e0999e2779d4f9 as hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740/ns/428a70b29de1472982e0999e2779d4f9 2024-11-14T14:54:57,637 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740/ns/428a70b29de1472982e0999e2779d4f9, entries=2, sequenceid=11, filesize=5.0 K 2024-11-14T14:54:57,638 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740/.tmp/table/3bd71f1b31374a54baa414b8add50fcb as hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740/table/3bd71f1b31374a54baa414b8add50fcb 2024-11-14T14:54:57,644 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740/table/3bd71f1b31374a54baa414b8add50fcb, entries=2, sequenceid=11, filesize=5.3 K 2024-11-14T14:54:57,645 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 146ms, sequenceid=11, compaction requested=false 2024-11-14T14:54:57,651 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-14T14:54:57,652 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:54:57,652 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:54:57,652 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731596097498Running coprocessor pre-close hooks at 1731596097498Disabling compacts and flushes for region at 1731596097498Disabling writes for close at 1731596097499 (+1 ms)Obtaining lock to block concurrent updates at 1731596097499Preparing flush snapshotting stores in 1588230740 at 1731596097499Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1731596097499Flushing stores of hbase:meta,,1.1588230740 at 1731596097516 (+17 ms)Flushing 1588230740/info: creating writer at 1731596097516Flushing 1588230740/info: appending metadata at 1731596097531 (+15 ms)Flushing 1588230740/info: closing flushed file at 1731596097531Flushing 1588230740/ns: creating writer at 1731596097554 (+23 ms)Flushing 1588230740/ns: appending metadata at 1731596097569 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731596097569Flushing 1588230740/table: creating writer at 1731596097584 (+15 ms)Flushing 1588230740/table: appending metadata at 1731596097606 (+22 ms)Flushing 1588230740/table: closing flushed file at 1731596097606Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2601b0b9: reopening flushed file at 1731596097623 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6222002b: reopening flushed file at 1731596097630 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2792cd41: reopening flushed file at 1731596097637 (+7 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 146ms, sequenceid=11, compaction requested=false at 1731596097645 (+8 ms)Writing region close event to WAL at 1731596097646 (+1 ms)Running coprocessor post-close hooks at 1731596097651 (+5 ms)Closed at 1731596097652 (+1 ms) 2024-11-14T14:54:57,652 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T14:54:57,663 INFO [regionserver/da1c6afcd1f9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T14:54:57,663 INFO [regionserver/da1c6afcd1f9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T14:54:57,698 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.HRegionServer(976): stopping server da1c6afcd1f9,40527,1731596059482; all regions closed. 2024-11-14T14:54:57,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:54:57,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38639 is added to blk_1073741901_1086 (size=825) 2024-11-14T14:54:57,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741901_1086 (size=825) 2024-11-14T14:54:57,877 INFO [regionserver/da1c6afcd1f9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:54:57,950 INFO [regionserver/da1c6afcd1f9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T14:54:57,950 INFO [regionserver/da1c6afcd1f9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T14:54:58,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38639 is added to blk_1073741877_1060 (size=12911) 2024-11-14T14:54:58,665 INFO [regionserver/da1c6afcd1f9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:54:59,616 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-11-14T14:54:59,617 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:54:59,617 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T14:55:00,823 INFO [master/da1c6afcd1f9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-14T14:55:00,823 INFO [master/da1c6afcd1f9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-14T14:55:01,501 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 after 4002ms 2024-11-14T14:55:01,517 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta after 4001ms 2024-11-14T14:55:01,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741835_1011 (size=393) 2024-11-14T14:55:01,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:55:02,499 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-11-14T14:55:02,502 DEBUG [RS:1;da1c6afcd1f9:34075 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/oldWALs 2024-11-14T14:55:02,502 INFO [RS:1;da1c6afcd1f9:34075 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog da1c6afcd1f9%2C34075%2C1731596060544:(num 1731596060811) 2024-11-14T14:55:02,502 DEBUG [RS:1;da1c6afcd1f9:34075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:55:02,502 INFO [RS:1;da1c6afcd1f9:34075 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:55:02,502 INFO [RS:1;da1c6afcd1f9:34075 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:55:02,502 INFO [RS:1;da1c6afcd1f9:34075 {}] hbase.ChoreService(370): Chore service for: regionserver/da1c6afcd1f9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T14:55:02,502 INFO [RS:1;da1c6afcd1f9:34075 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T14:55:02,502 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:55:02,502 INFO [RS:1;da1c6afcd1f9:34075 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T14:55:02,502 INFO [RS:1;da1c6afcd1f9:34075 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T14:55:02,502 INFO [RS:1;da1c6afcd1f9:34075 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:55:02,503 INFO [RS:1;da1c6afcd1f9:34075 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34075 2024-11-14T14:55:02,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34075-0x101a74678630002, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/da1c6afcd1f9,34075,1731596060544 2024-11-14T14:55:02,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:55:02,505 INFO [RS:1;da1c6afcd1f9:34075 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:55:02,507 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [da1c6afcd1f9,34075,1731596060544] 2024-11-14T14:55:02,507 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:02,509 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/da1c6afcd1f9,34075,1731596060544 already deleted, retry=false 2024-11-14T14:55:02,509 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; da1c6afcd1f9,34075,1731596060544 expired; onlineServers=1 2024-11-14T14:55:02,520 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:02,534 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:02,534 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:02,534 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:02,535 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:02,535 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:02,543 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:02,544 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:02,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34075-0x101a74678630002, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:55:02,607 INFO [RS:1;da1c6afcd1f9:34075 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:55:02,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34075-0x101a74678630002, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:55:02,607 INFO [RS:1;da1c6afcd1f9:34075 {}] regionserver.HRegionServer(1031): Exiting; stopping=da1c6afcd1f9,34075,1731596060544; zookeeper connection closed. 2024-11-14T14:55:02,608 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2af6090d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2af6090d 2024-11-14T14:55:02,700 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-11-14T14:55:02,704 DEBUG [RS:0;da1c6afcd1f9:40527 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/oldWALs 2024-11-14T14:55:02,704 INFO [RS:0;da1c6afcd1f9:40527 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog da1c6afcd1f9%2C40527%2C1731596059482.meta:.meta(num 1731596097500) 2024-11-14T14:55:02,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:02,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:02,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:02,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:02,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:02,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741891_1075 (size=16308) 2024-11-14T14:55:02,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38639 is added to blk_1073741891_1075 (size=16308) 2024-11-14T14:55:02,710 DEBUG [RS:0;da1c6afcd1f9:40527 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/oldWALs 2024-11-14T14:55:02,711 INFO [RS:0;da1c6afcd1f9:40527 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog da1c6afcd1f9%2C40527%2C1731596059482:(num 1731596097049) 2024-11-14T14:55:02,711 DEBUG [RS:0;da1c6afcd1f9:40527 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:55:02,711 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:55:02,711 INFO [RS:0;da1c6afcd1f9:40527 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:55:02,711 INFO [RS:0;da1c6afcd1f9:40527 {}] hbase.ChoreService(370): Chore service for: regionserver/da1c6afcd1f9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T14:55:02,711 INFO [RS:0;da1c6afcd1f9:40527 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:55:02,711 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:55:02,711 INFO [RS:0;da1c6afcd1f9:40527 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40527 2024-11-14T14:55:02,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/da1c6afcd1f9,40527,1731596059482 2024-11-14T14:55:02,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:55:02,714 INFO [RS:0;da1c6afcd1f9:40527 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:55:02,716 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [da1c6afcd1f9,40527,1731596059482] 2024-11-14T14:55:02,718 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/da1c6afcd1f9,40527,1731596059482 already deleted, retry=false 2024-11-14T14:55:02,718 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; da1c6afcd1f9,40527,1731596059482 expired; onlineServers=0 2024-11-14T14:55:02,718 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'da1c6afcd1f9,40173,1731596059405' ***** 2024-11-14T14:55:02,718 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T14:55:02,718 INFO [M:0;da1c6afcd1f9:40173 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:55:02,718 INFO [M:0;da1c6afcd1f9:40173 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:55:02,718 DEBUG [M:0;da1c6afcd1f9:40173 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T14:55:02,719 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T14:55:02,719 DEBUG [M:0;da1c6afcd1f9:40173 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T14:55:02,719 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731596059772 {}] cleaner.HFileCleaner(306): Exit Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731596059772,5,FailOnTimeoutGroup] 2024-11-14T14:55:02,719 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731596059764 {}] cleaner.HFileCleaner(306): Exit Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731596059764,5,FailOnTimeoutGroup] 2024-11-14T14:55:02,719 INFO [M:0;da1c6afcd1f9:40173 {}] hbase.ChoreService(370): Chore service for: master/da1c6afcd1f9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T14:55:02,719 INFO [M:0;da1c6afcd1f9:40173 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:55:02,719 DEBUG [M:0;da1c6afcd1f9:40173 {}] master.HMaster(1795): Stopping service threads 2024-11-14T14:55:02,719 INFO [M:0;da1c6afcd1f9:40173 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T14:55:02,719 INFO [M:0;da1c6afcd1f9:40173 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:55:02,720 INFO [M:0;da1c6afcd1f9:40173 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:55:02,720 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T14:55:02,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T14:55:02,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:02,720 DEBUG [M:0;da1c6afcd1f9:40173 {}] zookeeper.ZKUtil(347): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T14:55:02,721 WARN [M:0;da1c6afcd1f9:40173 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T14:55:02,721 INFO [M:0;da1c6afcd1f9:40173 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/.lastflushedseqids 2024-11-14T14:55:02,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741911_1097 (size=130) 2024-11-14T14:55:02,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38639 is added to blk_1073741911_1097 (size=130) 2024-11-14T14:55:02,730 INFO [M:0;da1c6afcd1f9:40173 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:55:02,730 INFO [M:0;da1c6afcd1f9:40173 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T14:55:02,730 DEBUG [M:0;da1c6afcd1f9:40173 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:55:02,730 INFO [M:0;da1c6afcd1f9:40173 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:55:02,730 DEBUG [M:0;da1c6afcd1f9:40173 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:55:02,730 DEBUG [M:0;da1c6afcd1f9:40173 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:55:02,730 DEBUG [M:0;da1c6afcd1f9:40173 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:55:02,730 INFO [M:0;da1c6afcd1f9:40173 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.26 KB heapSize=29.50 KB 2024-11-14T14:55:02,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:55:02,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:55:02,753 DEBUG [M:0;da1c6afcd1f9:40173 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7c56ee2e4e9147fda033db0ecdbb8c7a is 82, key is hbase:meta,,1/info:regioninfo/1731596060428/Put/seqid=0 2024-11-14T14:55:02,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38639 is added to blk_1073741912_1098 (size=5672) 2024-11-14T14:55:02,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741912_1098 (size=5672) 2024-11-14T14:55:02,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:55:02,816 INFO [RS:0;da1c6afcd1f9:40527 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:55:02,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40527-0x101a74678630001, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:55:02,816 INFO [RS:0;da1c6afcd1f9:40527 {}] regionserver.HRegionServer(1031): Exiting; stopping=da1c6afcd1f9,40527,1731596059482; zookeeper connection closed. 2024-11-14T14:55:02,817 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@65d4b1fd {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@65d4b1fd 2024-11-14T14:55:02,817 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-14T14:55:03,047 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-14T14:55:03,068 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:03,068 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:03,069 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:03,069 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:03,070 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:03,070 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:03,076 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:03,079 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:03,164 INFO [M:0;da1c6afcd1f9:40173 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7c56ee2e4e9147fda033db0ecdbb8c7a 2024-11-14T14:55:03,188 DEBUG [M:0;da1c6afcd1f9:40173 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/75238ac969f24f1981b022faba2dfc15 is 775, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731596061130/Put/seqid=0 2024-11-14T14:55:03,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741913_1099 (size=6256) 2024-11-14T14:55:03,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38639 is added to blk_1073741913_1099 (size=6256) 2024-11-14T14:55:03,199 INFO [M:0;da1c6afcd1f9:40173 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.59 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/75238ac969f24f1981b022faba2dfc15 2024-11-14T14:55:03,206 INFO [M:0;da1c6afcd1f9:40173 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 75238ac969f24f1981b022faba2dfc15 2024-11-14T14:55:03,223 DEBUG [M:0;da1c6afcd1f9:40173 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/74212f10a1ca44dc8d4dc463c8da6be8 is 69, key is da1c6afcd1f9,34075,1731596060544/rs:state/1731596060617/Put/seqid=0 2024-11-14T14:55:03,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38639 is added to blk_1073741914_1100 (size=5224) 2024-11-14T14:55:03,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741914_1100 (size=5224) 2024-11-14T14:55:03,231 INFO [M:0;da1c6afcd1f9:40173 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/74212f10a1ca44dc8d4dc463c8da6be8 2024-11-14T14:55:03,255 DEBUG [M:0;da1c6afcd1f9:40173 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/398f2715cd754cb2854b0138706c78ad is 52, key is load_balancer_on/state:d/1731596060525/Put/seqid=0 2024-11-14T14:55:03,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38639 is added to blk_1073741915_1101 (size=5056) 2024-11-14T14:55:03,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741915_1101 (size=5056) 2024-11-14T14:55:03,264 INFO [M:0;da1c6afcd1f9:40173 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/398f2715cd754cb2854b0138706c78ad 2024-11-14T14:55:03,271 DEBUG [M:0;da1c6afcd1f9:40173 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7c56ee2e4e9147fda033db0ecdbb8c7a as hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7c56ee2e4e9147fda033db0ecdbb8c7a 2024-11-14T14:55:03,278 INFO [M:0;da1c6afcd1f9:40173 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7c56ee2e4e9147fda033db0ecdbb8c7a, entries=8, sequenceid=60, filesize=5.5 K 2024-11-14T14:55:03,279 DEBUG [M:0;da1c6afcd1f9:40173 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/75238ac969f24f1981b022faba2dfc15 as hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/75238ac969f24f1981b022faba2dfc15 2024-11-14T14:55:03,285 INFO [M:0;da1c6afcd1f9:40173 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 75238ac969f24f1981b022faba2dfc15 2024-11-14T14:55:03,286 INFO [M:0;da1c6afcd1f9:40173 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/75238ac969f24f1981b022faba2dfc15, entries=6, sequenceid=60, filesize=6.1 K 2024-11-14T14:55:03,287 DEBUG [M:0;da1c6afcd1f9:40173 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/74212f10a1ca44dc8d4dc463c8da6be8 as hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/74212f10a1ca44dc8d4dc463c8da6be8 2024-11-14T14:55:03,293 INFO [M:0;da1c6afcd1f9:40173 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/74212f10a1ca44dc8d4dc463c8da6be8, entries=2, sequenceid=60, filesize=5.1 K 2024-11-14T14:55:03,294 DEBUG [M:0;da1c6afcd1f9:40173 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/398f2715cd754cb2854b0138706c78ad as hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/398f2715cd754cb2854b0138706c78ad 2024-11-14T14:55:03,301 INFO [M:0;da1c6afcd1f9:40173 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/398f2715cd754cb2854b0138706c78ad, entries=1, sequenceid=60, filesize=4.9 K 2024-11-14T14:55:03,302 INFO [M:0;da1c6afcd1f9:40173 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 572ms, sequenceid=60, compaction requested=false 2024-11-14T14:55:03,304 INFO [M:0;da1c6afcd1f9:40173 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:55:03,305 DEBUG [M:0;da1c6afcd1f9:40173 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731596102730Disabling compacts and flushes for region at 1731596102730Disabling writes for close at 1731596102730Obtaining lock to block concurrent updates at 1731596102730Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731596102730Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23817, getHeapSize=30144, getOffHeapSize=0, getCellsCount=71 at 1731596102731 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731596102732 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731596102732Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731596102752 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731596102752Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731596103171 (+419 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731596103187 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731596103187Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731596103206 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731596103222 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731596103222Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731596103237 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731596103254 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731596103254Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3193f67: reopening flushed file at 1731596103270 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@575f5d9d: reopening flushed file at 1731596103278 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1118c7d8: reopening flushed file at 1731596103286 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34d82c57: reopening flushed file at 1731596103293 (+7 ms)Finished flush of dataSize ~23.26 KB/23817, heapSize ~29.44 KB/30144, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 572ms, sequenceid=60, compaction requested=false at 1731596103302 (+9 ms)Writing region close event to WAL at 1731596103304 (+2 ms)Closed at 1731596103304 2024-11-14T14:55:03,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:03,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:03,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:03,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:03,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:03,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38639 is added to blk_1073741888_1071 (size=1045) 2024-11-14T14:55:03,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741888_1071 (size=1045) 2024-11-14T14:55:03,508 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:03,518 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:03,757 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@44344c36 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1040518630-172.17.0.2-1731596058577:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:33793,null,null]) java.net.ConnectException: Call From da1c6afcd1f9/172.17.0.2 to localhost:43867 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-14T14:55:04,509 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:04,518 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:04,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:55:04,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44787 is added to blk_1073741836_1012 (size=76) 2024-11-14T14:55:04,751 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/WALs/da1c6afcd1f9,40173,1731596059405/da1c6afcd1f9%2C40173%2C1731596059405.1731596059635 to hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/oldWALs/da1c6afcd1f9%2C40173%2C1731596059405.1731596059635 2024-11-14T14:55:04,755 INFO [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/MasterData/oldWALs/da1c6afcd1f9%2C40173%2C1731596059405.1731596059635 to hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/oldWALs/da1c6afcd1f9%2C40173%2C1731596059405.1731596059635$masterlocalwal$ 2024-11-14T14:55:04,755 INFO [M:0;da1c6afcd1f9:40173 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T14:55:04,755 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:55:04,755 INFO [M:0;da1c6afcd1f9:40173 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40173 2024-11-14T14:55:04,756 INFO [M:0;da1c6afcd1f9:40173 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:55:04,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:55:04,858 INFO [M:0;da1c6afcd1f9:40173 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:55:04,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40173-0x101a74678630000, quorum=127.0.0.1:60317, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:55:04,861 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5536ce0c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:55:04,862 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4c03d0a2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:55:04,862 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:55:04,862 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@719d00c0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:55:04,862 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2aaad1eb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/hadoop.log.dir/,STOPPED} 2024-11-14T14:55:04,864 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@3c0e4cba {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1040518630-172.17.0.2-1731596058577:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:33793,null,null]) java.io.InterruptedIOException: DestHost:destPort localhost:43867 , LocalHost:localPort da1c6afcd1f9/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-14T14:55:04,865 WARN [BP-1040518630-172.17.0.2-1731596058577 heartbeating to localhost/127.0.0.1:35595 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1040518630-172.17.0.2-1731596058577 (Datanode Uuid ed0576a8-2ece-4054-b155-ea617541c969) service to localhost/127.0.0.1:35595 2024-11-14T14:55:04,866 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data3/current/BP-1040518630-172.17.0.2-1731596058577 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:55:04,866 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:55:04,866 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data4/current/BP-1040518630-172.17.0.2-1731596058577 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:55:04,866 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@3c0e4cba {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1040518630-172.17.0.2-1731596058577:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:38639,null,null]) java.io.IOException: No block pool offer service for bpid=BP-1040518630-172.17.0.2-1731596058577 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:04,866 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@3c0e4cba {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1040518630-172.17.0.2-1731596058577:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:33793,null,null]) java.io.IOException: No block pool offer service for bpid=BP-1040518630-172.17.0.2-1731596058577 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:04,866 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@3c0e4cba {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1040518630-172.17.0.2-1731596058577:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:38639,null,null], DatanodeInfoWithStorage[127.0.0.1:33793,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1040518630-172.17.0.2-1731596058577:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:38639,null,null], DatanodeInfoWithStorage[127.0.0.1:33793,null,null]] 2024-11-14T14:55:04,868 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@28246fba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:55:04,869 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5056747b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:55:04,869 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:55:04,869 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@68004957{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:55:04,869 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1dc59954{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/hadoop.log.dir/,STOPPED} 2024-11-14T14:55:04,872 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:55:04,872 WARN [BP-1040518630-172.17.0.2-1731596058577 heartbeating to localhost/127.0.0.1:35595 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:55:04,872 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:55:04,872 WARN [BP-1040518630-172.17.0.2-1731596058577 heartbeating to localhost/127.0.0.1:35595 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1040518630-172.17.0.2-1731596058577 (Datanode Uuid 3961c212-814b-45b5-8059-b2afc9c559ae) service to localhost/127.0.0.1:35595 2024-11-14T14:55:04,873 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data9/current/BP-1040518630-172.17.0.2-1731596058577 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:55:04,873 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/cluster_3fec5121-1e5a-6cb8-6ded-fbfe89dd60e1/data/data10/current/BP-1040518630-172.17.0.2-1731596058577 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:55:04,874 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:55:04,881 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@195100a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T14:55:04,881 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1e349773{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:55:04,881 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:55:04,882 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6a488aac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:55:04,882 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2305029e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/hadoop.log.dir/,STOPPED} 2024-11-14T14:55:04,891 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T14:55:04,928 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-14T14:55:04,942 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=157 (was 81) Potentially hanging thread: LeaseRenewer:jenkins@localhost:42069 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f7a88bf4ff8.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35595 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-18-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35595 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:35595 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35595 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:35595 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35595 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Command processor java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1411) app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35595 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-18-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'DataNode' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35595 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-19-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-18-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:42069 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35595 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:35595 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35595 from jenkins.hfs.3 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-20-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-19-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f7a88bf4ff8.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-19-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=450 (was 405) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=115 (was 124), ProcessCount=11 (was 11), AvailableMemoryMB=7727 (was 9067) 2024-11-14T14:55:04,952 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=157, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=115, ProcessCount=11, AvailableMemoryMB=7727 2024-11-14T14:55:04,953 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T14:55:04,953 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/hadoop.log.dir so I do NOT create it in target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c 2024-11-14T14:55:04,953 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/79962286-a5ae-a3a5-eefb-724b738788ea/hadoop.tmp.dir so I do NOT create it in target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c 2024-11-14T14:55:04,953 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36, deleteOnExit=true 2024-11-14T14:55:04,953 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T14:55:04,953 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/test.cache.data in system properties and HBase conf 2024-11-14T14:55:04,954 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T14:55:04,954 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/hadoop.log.dir in system properties and HBase conf 2024-11-14T14:55:04,954 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T14:55:04,954 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T14:55:04,954 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T14:55:04,954 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T14:55:04,954 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T14:55:04,954 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T14:55:04,955 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T14:55:04,955 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T14:55:04,955 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T14:55:04,955 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T14:55:04,955 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T14:55:04,955 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T14:55:04,955 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T14:55:04,955 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/nfs.dump.dir in system properties and HBase conf 2024-11-14T14:55:04,955 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/java.io.tmpdir in system properties and HBase conf 2024-11-14T14:55:04,955 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T14:55:04,956 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T14:55:04,956 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T14:55:04,976 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-14T14:55:05,053 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:55:05,060 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:55:05,062 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:55:05,062 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:55:05,062 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T14:55:05,063 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:55:05,064 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13fdd007{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:55:05,064 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@10a92d53{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:55:05,196 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@ba4770c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/java.io.tmpdir/jetty-localhost-35373-hadoop-hdfs-3_4_1-tests_jar-_-any-92695435168312016/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T14:55:05,197 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4702e786{HTTP/1.1, (http/1.1)}{localhost:35373} 2024-11-14T14:55:05,197 INFO [Time-limited test {}] server.Server(415): Started @150819ms 2024-11-14T14:55:05,212 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-14T14:55:05,327 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:55:05,331 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:55:05,336 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:55:05,336 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:55:05,336 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T14:55:05,337 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74ea1d44{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:55:05,337 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33255ae1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:55:05,458 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@45628471{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/java.io.tmpdir/jetty-localhost-40771-hadoop-hdfs-3_4_1-tests_jar-_-any-13889204146874650881/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:55:05,458 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4e6129e0{HTTP/1.1, (http/1.1)}{localhost:40771} 2024-11-14T14:55:05,459 INFO [Time-limited test {}] server.Server(415): Started @151080ms 2024-11-14T14:55:05,461 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:55:05,506 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:55:05,510 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:55:05,510 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:05,515 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:55:05,515 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:55:05,515 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:55:05,516 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4360f0f4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:55:05,517 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@586e8021{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:55:05,519 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:05,573 WARN [Thread-1199 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/data/data1/current/BP-517382180-172.17.0.2-1731596104998/current, will proceed with Du for space computation calculation, 2024-11-14T14:55:05,573 WARN [Thread-1200 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/data/data2/current/BP-517382180-172.17.0.2-1731596104998/current, will proceed with Du for space computation calculation, 2024-11-14T14:55:05,599 WARN [Thread-1178 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:55:05,603 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1cb417bcd1dfa0dc with lease ID 0x67fbf7156c0a9282: Processing first storage report for DS-42e12750-ea13-4781-9ab0-d5facb94aa61 from datanode DatanodeRegistration(127.0.0.1:42517, datanodeUuid=972bbbb2-749b-46dd-bc90-7ff3b222239e, infoPort=39091, infoSecurePort=0, ipcPort=38813, storageInfo=lv=-57;cid=testClusterID;nsid=1672118275;c=1731596104998) 2024-11-14T14:55:05,603 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1cb417bcd1dfa0dc with lease ID 0x67fbf7156c0a9282: from storage DS-42e12750-ea13-4781-9ab0-d5facb94aa61 node DatanodeRegistration(127.0.0.1:42517, datanodeUuid=972bbbb2-749b-46dd-bc90-7ff3b222239e, infoPort=39091, infoSecurePort=0, ipcPort=38813, storageInfo=lv=-57;cid=testClusterID;nsid=1672118275;c=1731596104998), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T14:55:05,603 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1cb417bcd1dfa0dc with lease ID 0x67fbf7156c0a9282: Processing first storage report for DS-790acd79-c81c-4653-b697-859334b8bfee from datanode DatanodeRegistration(127.0.0.1:42517, datanodeUuid=972bbbb2-749b-46dd-bc90-7ff3b222239e, infoPort=39091, infoSecurePort=0, ipcPort=38813, storageInfo=lv=-57;cid=testClusterID;nsid=1672118275;c=1731596104998) 2024-11-14T14:55:05,603 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1cb417bcd1dfa0dc with lease ID 0x67fbf7156c0a9282: from storage DS-790acd79-c81c-4653-b697-859334b8bfee node DatanodeRegistration(127.0.0.1:42517, datanodeUuid=972bbbb2-749b-46dd-bc90-7ff3b222239e, infoPort=39091, infoSecurePort=0, ipcPort=38813, storageInfo=lv=-57;cid=testClusterID;nsid=1672118275;c=1731596104998), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:55:05,652 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4a7cb65f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/java.io.tmpdir/jetty-localhost-44117-hadoop-hdfs-3_4_1-tests_jar-_-any-4747696298278819885/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:55:05,652 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@49d720be{HTTP/1.1, (http/1.1)}{localhost:44117} 2024-11-14T14:55:05,652 INFO [Time-limited test {}] server.Server(415): Started @151274ms 2024-11-14T14:55:05,654 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:55:05,760 WARN [Thread-1225 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/data/data3/current/BP-517382180-172.17.0.2-1731596104998/current, will proceed with Du for space computation calculation, 2024-11-14T14:55:05,760 WARN [Thread-1226 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/data/data4/current/BP-517382180-172.17.0.2-1731596104998/current, will proceed with Du for space computation calculation, 2024-11-14T14:55:05,787 WARN [Thread-1214 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:55:05,789 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa14944dba57253da with lease ID 0x67fbf7156c0a9283: Processing first storage report for DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b from datanode DatanodeRegistration(127.0.0.1:39777, datanodeUuid=1d51cfb9-dd20-4689-a9a3-b54aa021f596, infoPort=34705, infoSecurePort=0, ipcPort=37299, storageInfo=lv=-57;cid=testClusterID;nsid=1672118275;c=1731596104998) 2024-11-14T14:55:05,790 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa14944dba57253da with lease ID 0x67fbf7156c0a9283: from storage DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b node DatanodeRegistration(127.0.0.1:39777, datanodeUuid=1d51cfb9-dd20-4689-a9a3-b54aa021f596, infoPort=34705, infoSecurePort=0, ipcPort=37299, storageInfo=lv=-57;cid=testClusterID;nsid=1672118275;c=1731596104998), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:55:05,790 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa14944dba57253da with lease ID 0x67fbf7156c0a9283: Processing first storage report for DS-9f179d08-6877-4364-bb7b-916f3d937328 from datanode DatanodeRegistration(127.0.0.1:39777, datanodeUuid=1d51cfb9-dd20-4689-a9a3-b54aa021f596, infoPort=34705, infoSecurePort=0, ipcPort=37299, storageInfo=lv=-57;cid=testClusterID;nsid=1672118275;c=1731596104998) 2024-11-14T14:55:05,790 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa14944dba57253da with lease ID 0x67fbf7156c0a9283: from storage DS-9f179d08-6877-4364-bb7b-916f3d937328 node DatanodeRegistration(127.0.0.1:39777, datanodeUuid=1d51cfb9-dd20-4689-a9a3-b54aa021f596, infoPort=34705, infoSecurePort=0, ipcPort=37299, storageInfo=lv=-57;cid=testClusterID;nsid=1672118275;c=1731596104998), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:55:05,888 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c 2024-11-14T14:55:05,892 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/zookeeper_0, clientPort=63452, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T14:55:05,897 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63452 2024-11-14T14:55:05,897 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:55:05,899 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:55:05,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42517 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:55:05,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39777 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:55:05,929 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de with version=8 2024-11-14T14:55:05,930 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/hbase-staging 2024-11-14T14:55:05,933 INFO [Time-limited test {}] client.ConnectionUtils(128): master/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:55:05,933 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:55:05,933 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:55:05,933 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:55:05,933 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:55:05,933 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:55:05,933 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T14:55:05,934 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:55:05,935 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45313 2024-11-14T14:55:05,936 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45313 connecting to ZooKeeper ensemble=127.0.0.1:63452 2024-11-14T14:55:05,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:453130x0, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:55:05,956 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45313-0x101a7472e0b0000 connected 2024-11-14T14:55:06,006 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:55:06,008 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:55:06,013 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:55:06,013 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de, hbase.cluster.distributed=false 2024-11-14T14:55:06,015 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:55:06,024 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45313 2024-11-14T14:55:06,028 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45313 2024-11-14T14:55:06,036 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45313 2024-11-14T14:55:06,052 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45313 2024-11-14T14:55:06,052 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45313 2024-11-14T14:55:06,071 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:55:06,071 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:55:06,072 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:55:06,072 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:55:06,072 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:55:06,072 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:55:06,072 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T14:55:06,072 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:55:06,074 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39339 2024-11-14T14:55:06,076 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39339 connecting to ZooKeeper ensemble=127.0.0.1:63452 2024-11-14T14:55:06,077 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:55:06,079 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:55:06,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:393390x0, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:55:06,093 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:393390x0, quorum=127.0.0.1:63452, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:55:06,093 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T14:55:06,100 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39339-0x101a7472e0b0001 connected 2024-11-14T14:55:06,116 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T14:55:06,117 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T14:55:06,119 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:55:06,124 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39339 2024-11-14T14:55:06,128 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39339 2024-11-14T14:55:06,140 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39339 2024-11-14T14:55:06,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39339 2024-11-14T14:55:06,160 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39339 2024-11-14T14:55:06,177 DEBUG [M:0;da1c6afcd1f9:45313 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;da1c6afcd1f9:45313 2024-11-14T14:55:06,178 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/da1c6afcd1f9,45313,1731596105932 2024-11-14T14:55:06,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:55:06,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:55:06,181 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/da1c6afcd1f9,45313,1731596105932 2024-11-14T14:55:06,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T14:55:06,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:06,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:06,185 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T14:55:06,185 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/da1c6afcd1f9,45313,1731596105932 from backup master directory 2024-11-14T14:55:06,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/da1c6afcd1f9,45313,1731596105932 2024-11-14T14:55:06,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:55:06,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:55:06,187 WARN [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:55:06,187 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=da1c6afcd1f9,45313,1731596105932 2024-11-14T14:55:06,202 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/hbase.id] with ID: 20d642ec-f49f-4613-8218-38b819f829d9 2024-11-14T14:55:06,202 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/.tmp/hbase.id 2024-11-14T14:55:06,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39777 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:55:06,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42517 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:55:06,213 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/.tmp/hbase.id]:[hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/hbase.id] 2024-11-14T14:55:06,228 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:55:06,228 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T14:55:06,230 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-14T14:55:06,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:06,232 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:06,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39777 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:55:06,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42517 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:55:06,246 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T14:55:06,246 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T14:55:06,247 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:55:06,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42517 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:55:06,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39777 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:55:06,510 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:06,519 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:06,662 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store 2024-11-14T14:55:06,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42517 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:55:06,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39777 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:55:06,671 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:55:06,671 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:55:06,671 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:55:06,671 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:55:06,671 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:55:06,671 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:55:06,671 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:55:06,671 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731596106671Disabling compacts and flushes for region at 1731596106671Disabling writes for close at 1731596106671Writing region close event to WAL at 1731596106671Closed at 1731596106671 2024-11-14T14:55:06,672 WARN [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/.initializing 2024-11-14T14:55:06,672 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/WALs/da1c6afcd1f9,45313,1731596105932 2024-11-14T14:55:06,675 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C45313%2C1731596105932, suffix=, logDir=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/WALs/da1c6afcd1f9,45313,1731596105932, archiveDir=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/oldWALs, maxLogs=10 2024-11-14T14:55:06,676 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C45313%2C1731596105932.1731596106675 2024-11-14T14:55:06,681 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/WALs/da1c6afcd1f9,45313,1731596105932/da1c6afcd1f9%2C45313%2C1731596105932.1731596106675 2024-11-14T14:55:06,682 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39091:39091),(127.0.0.1/127.0.0.1:34705:34705)] 2024-11-14T14:55:06,683 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:55:06,683 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:55:06,683 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:06,683 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:06,688 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:06,690 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T14:55:06,690 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:06,690 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:55:06,690 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:06,692 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T14:55:06,692 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:06,693 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:55:06,693 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:06,695 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T14:55:06,695 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:06,696 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:55:06,696 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:06,697 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T14:55:06,697 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:06,698 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:55:06,698 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:06,698 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:06,699 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:06,700 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:06,700 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:06,701 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T14:55:06,712 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:06,715 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:55:06,716 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=879515, jitterRate=0.11836178600788116}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T14:55:06,717 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731596106683Initializing all the Stores at 1731596106684 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596106684Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596106688 (+4 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596106688Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596106688Cleaning up temporary data from old regions at 1731596106701 (+13 ms)Region opened successfully at 1731596106717 (+16 ms) 2024-11-14T14:55:06,717 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T14:55:06,725 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@270c9a0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:55:06,727 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T14:55:06,727 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T14:55:06,727 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T14:55:06,727 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:55:06,728 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:55:06,728 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-14T14:55:06,728 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T14:55:06,734 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T14:55:06,735 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T14:55:06,737 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T14:55:06,738 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T14:55:06,739 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T14:55:06,741 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T14:55:06,741 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T14:55:06,742 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T14:55:06,745 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T14:55:06,748 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T14:55:06,750 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T14:55:06,753 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T14:55:06,755 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T14:55:06,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:55:06,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:55:06,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:06,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:06,759 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=da1c6afcd1f9,45313,1731596105932, sessionid=0x101a7472e0b0000, setting cluster-up flag (Was=false) 2024-11-14T14:55:06,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:06,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:06,771 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T14:55:06,773 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da1c6afcd1f9,45313,1731596105932 2024-11-14T14:55:06,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:06,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:06,785 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T14:55:06,786 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da1c6afcd1f9,45313,1731596105932 2024-11-14T14:55:06,788 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T14:55:06,794 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:55:06,795 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T14:55:06,795 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T14:55:06,795 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: da1c6afcd1f9,45313,1731596105932 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T14:55:06,798 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:55:06,798 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:55:06,798 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:55:06,798 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:55:06,798 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/da1c6afcd1f9:0, corePoolSize=10, maxPoolSize=10 2024-11-14T14:55:06,798 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:06,798 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:55:06,798 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:06,811 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:55:06,811 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T14:55:06,812 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:06,813 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T14:55:06,824 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731596136824 2024-11-14T14:55:06,824 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T14:55:06,824 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T14:55:06,824 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T14:55:06,824 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T14:55:06,824 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T14:55:06,824 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T14:55:06,828 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:06,836 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T14:55:06,836 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T14:55:06,836 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T14:55:06,840 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T14:55:06,840 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T14:55:06,848 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731596106840,5,FailOnTimeoutGroup] 2024-11-14T14:55:06,849 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731596106848,5,FailOnTimeoutGroup] 2024-11-14T14:55:06,849 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:06,850 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T14:55:06,850 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:06,850 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:06,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39777 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:55:06,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42517 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:55:06,858 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T14:55:06,858 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de 2024-11-14T14:55:06,873 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.HRegionServer(746): ClusterId : 20d642ec-f49f-4613-8218-38b819f829d9 2024-11-14T14:55:06,874 DEBUG [RS:0;da1c6afcd1f9:39339 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T14:55:06,877 DEBUG [RS:0;da1c6afcd1f9:39339 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T14:55:06,877 DEBUG [RS:0;da1c6afcd1f9:39339 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T14:55:06,881 DEBUG [RS:0;da1c6afcd1f9:39339 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T14:55:06,881 DEBUG [RS:0;da1c6afcd1f9:39339 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f6c4576, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:55:06,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39777 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:55:06,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42517 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:55:06,899 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:55:06,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:55:06,903 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:55:06,903 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:06,904 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:55:06,904 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:55:06,904 DEBUG [RS:0;da1c6afcd1f9:39339 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;da1c6afcd1f9:39339 2024-11-14T14:55:06,904 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T14:55:06,904 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T14:55:06,904 DEBUG [RS:0;da1c6afcd1f9:39339 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T14:55:06,905 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.HRegionServer(2659): reportForDuty to master=da1c6afcd1f9,45313,1731596105932 with port=39339, startcode=1731596106071 2024-11-14T14:55:06,905 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:55:06,905 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:06,906 DEBUG [RS:0;da1c6afcd1f9:39339 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T14:55:06,906 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:55:06,906 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:55:06,907 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:55:06,908 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:06,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:55:06,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:55:06,908 INFO [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51589, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T14:55:06,909 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45313 {}] master.ServerManager(363): Checking decommissioned status of RegionServer da1c6afcd1f9,39339,1731596106071 2024-11-14T14:55:06,909 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45313 {}] master.ServerManager(517): Registering regionserver=da1c6afcd1f9,39339,1731596106071 2024-11-14T14:55:06,910 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:55:06,910 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:06,910 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:55:06,910 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:55:06,911 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740 2024-11-14T14:55:06,911 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740 2024-11-14T14:55:06,911 DEBUG [RS:0;da1c6afcd1f9:39339 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de 2024-11-14T14:55:06,911 DEBUG [RS:0;da1c6afcd1f9:39339 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33315 2024-11-14T14:55:06,911 DEBUG [RS:0;da1c6afcd1f9:39339 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T14:55:06,913 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:55:06,913 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:55:06,913 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T14:55:06,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:55:06,914 DEBUG [RS:0;da1c6afcd1f9:39339 {}] zookeeper.ZKUtil(111): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/da1c6afcd1f9,39339,1731596106071 2024-11-14T14:55:06,914 WARN [RS:0;da1c6afcd1f9:39339 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:55:06,914 INFO [RS:0;da1c6afcd1f9:39339 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:55:06,914 DEBUG [RS:0;da1c6afcd1f9:39339 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071 2024-11-14T14:55:06,915 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [da1c6afcd1f9,39339,1731596106071] 2024-11-14T14:55:06,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:55:06,917 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:55:06,918 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=859540, jitterRate=0.09296271204948425}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T14:55:06,919 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731596106899Initializing all the Stores at 1731596106900 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596106900Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596106901 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596106901Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596106901Cleaning up temporary data from old regions at 1731596106913 (+12 ms)Region opened successfully at 1731596106918 (+5 ms) 2024-11-14T14:55:06,919 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:55:06,919 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:55:06,919 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:55:06,919 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:55:06,919 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:55:06,919 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T14:55:06,919 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:55:06,919 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731596106919Disabling compacts and flushes for region at 1731596106919Disabling writes for close at 1731596106919Writing region close event to WAL at 1731596106919Closed at 1731596106919 2024-11-14T14:55:06,921 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:55:06,921 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T14:55:06,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T14:55:06,922 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:55:06,924 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T14:55:06,925 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T14:55:06,932 INFO [RS:0;da1c6afcd1f9:39339 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T14:55:06,932 INFO [RS:0;da1c6afcd1f9:39339 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:06,936 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T14:55:06,941 INFO [RS:0;da1c6afcd1f9:39339 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T14:55:06,941 INFO [RS:0;da1c6afcd1f9:39339 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:06,942 DEBUG [RS:0;da1c6afcd1f9:39339 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:06,942 DEBUG [RS:0;da1c6afcd1f9:39339 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:06,942 DEBUG [RS:0;da1c6afcd1f9:39339 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:06,942 DEBUG [RS:0;da1c6afcd1f9:39339 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:06,942 DEBUG [RS:0;da1c6afcd1f9:39339 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:06,942 DEBUG [RS:0;da1c6afcd1f9:39339 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:55:06,942 DEBUG [RS:0;da1c6afcd1f9:39339 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:06,942 DEBUG [RS:0;da1c6afcd1f9:39339 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:06,942 DEBUG [RS:0;da1c6afcd1f9:39339 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:06,942 DEBUG [RS:0;da1c6afcd1f9:39339 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:06,942 DEBUG [RS:0;da1c6afcd1f9:39339 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:06,942 DEBUG [RS:0;da1c6afcd1f9:39339 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:06,942 DEBUG [RS:0;da1c6afcd1f9:39339 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:55:06,942 DEBUG [RS:0;da1c6afcd1f9:39339 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:55:06,944 INFO [RS:0;da1c6afcd1f9:39339 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:06,944 INFO [RS:0;da1c6afcd1f9:39339 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:06,944 INFO [RS:0;da1c6afcd1f9:39339 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:06,944 INFO [RS:0;da1c6afcd1f9:39339 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:06,944 INFO [RS:0;da1c6afcd1f9:39339 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:06,944 INFO [RS:0;da1c6afcd1f9:39339 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,39339,1731596106071-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:55:06,973 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T14:55:06,973 INFO [RS:0;da1c6afcd1f9:39339 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,39339,1731596106071-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:06,973 INFO [RS:0;da1c6afcd1f9:39339 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:06,974 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.Replication(171): da1c6afcd1f9,39339,1731596106071 started 2024-11-14T14:55:06,992 INFO [RS:0;da1c6afcd1f9:39339 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:06,992 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.HRegionServer(1482): Serving as da1c6afcd1f9,39339,1731596106071, RpcServer on da1c6afcd1f9/172.17.0.2:39339, sessionid=0x101a7472e0b0001 2024-11-14T14:55:06,992 DEBUG [RS:0;da1c6afcd1f9:39339 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T14:55:06,992 DEBUG [RS:0;da1c6afcd1f9:39339 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager da1c6afcd1f9,39339,1731596106071 2024-11-14T14:55:06,992 DEBUG [RS:0;da1c6afcd1f9:39339 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,39339,1731596106071' 2024-11-14T14:55:06,992 DEBUG [RS:0;da1c6afcd1f9:39339 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T14:55:06,993 DEBUG [RS:0;da1c6afcd1f9:39339 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T14:55:06,994 DEBUG [RS:0;da1c6afcd1f9:39339 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T14:55:06,994 DEBUG [RS:0;da1c6afcd1f9:39339 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T14:55:06,994 DEBUG [RS:0;da1c6afcd1f9:39339 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager da1c6afcd1f9,39339,1731596106071 2024-11-14T14:55:06,994 DEBUG [RS:0;da1c6afcd1f9:39339 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,39339,1731596106071' 2024-11-14T14:55:06,994 DEBUG [RS:0;da1c6afcd1f9:39339 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T14:55:06,994 DEBUG [RS:0;da1c6afcd1f9:39339 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T14:55:06,995 DEBUG [RS:0;da1c6afcd1f9:39339 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T14:55:06,995 INFO [RS:0;da1c6afcd1f9:39339 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T14:55:06,995 INFO [RS:0;da1c6afcd1f9:39339 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T14:55:07,075 WARN [da1c6afcd1f9:45313 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-14T14:55:07,097 INFO [RS:0;da1c6afcd1f9:39339 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C39339%2C1731596106071, suffix=, logDir=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071, archiveDir=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/oldWALs, maxLogs=32 2024-11-14T14:55:07,098 INFO [RS:0;da1c6afcd1f9:39339 {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C39339%2C1731596106071.1731596107098 2024-11-14T14:55:07,106 INFO [RS:0;da1c6afcd1f9:39339 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596107098 2024-11-14T14:55:07,107 DEBUG [RS:0;da1c6afcd1f9:39339 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39091:39091),(127.0.0.1/127.0.0.1:34705:34705)] 2024-11-14T14:55:07,326 DEBUG [da1c6afcd1f9:45313 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T14:55:07,326 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=da1c6afcd1f9,39339,1731596106071 2024-11-14T14:55:07,328 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,39339,1731596106071, state=OPENING 2024-11-14T14:55:07,330 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T14:55:07,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:07,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:07,333 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:55:07,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:55:07,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,39339,1731596106071}] 2024-11-14T14:55:07,333 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:55:07,486 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T14:55:07,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34511, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T14:55:07,493 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T14:55:07,494 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:55:07,495 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C39339%2C1731596106071.meta, suffix=.meta, logDir=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071, archiveDir=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/oldWALs, maxLogs=32 2024-11-14T14:55:07,496 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C39339%2C1731596106071.meta.1731596107496.meta 2024-11-14T14:55:07,502 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.meta.1731596107496.meta 2024-11-14T14:55:07,502 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39091:39091),(127.0.0.1/127.0.0.1:34705:34705)] 2024-11-14T14:55:07,503 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:55:07,503 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T14:55:07,504 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T14:55:07,504 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T14:55:07,504 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T14:55:07,504 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:55:07,504 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T14:55:07,504 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T14:55:07,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:55:07,506 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:55:07,506 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:07,507 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:55:07,507 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:55:07,508 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:55:07,508 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:07,508 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:55:07,509 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:55:07,509 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:55:07,509 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:07,510 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:55:07,510 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:55:07,511 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:55:07,511 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:07,511 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:07,512 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:55:07,512 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:55:07,513 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740 2024-11-14T14:55:07,514 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740 2024-11-14T14:55:07,515 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:55:07,515 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:55:07,516 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T14:55:07,518 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:55:07,519 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=772262, jitterRate=-0.018018320202827454}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T14:55:07,519 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T14:55:07,520 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731596107504Writing region info on filesystem at 1731596107504Initializing all the Stores at 1731596107505 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596107505Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596107505Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596107505Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596107505Cleaning up temporary data from old regions at 1731596107515 (+10 ms)Running coprocessor post-open hooks at 1731596107519 (+4 ms)Region opened successfully at 1731596107520 (+1 ms) 2024-11-14T14:55:07,520 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:07,521 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731596107486 2024-11-14T14:55:07,524 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T14:55:07,524 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T14:55:07,525 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,39339,1731596106071 2024-11-14T14:55:07,526 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,39339,1731596106071, state=OPEN 2024-11-14T14:55:07,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:55:07,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:55:07,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:55:07,532 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,39339,1731596106071 2024-11-14T14:55:07,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:55:07,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T14:55:07,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,39339,1731596106071 in 199 msec 2024-11-14T14:55:07,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T14:55:07,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 615 msec 2024-11-14T14:55:07,540 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:55:07,540 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T14:55:07,542 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:55:07,542 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,39339,1731596106071, seqNum=-1] 2024-11-14T14:55:07,543 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:55:07,544 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48729, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:55:07,552 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 757 msec 2024-11-14T14:55:07,552 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731596107552, completionTime=-1 2024-11-14T14:55:07,552 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T14:55:07,552 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:55:07,555 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-14T14:55:07,555 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731596167555 2024-11-14T14:55:07,555 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731596227555 2024-11-14T14:55:07,555 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-14T14:55:07,555 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,45313,1731596105932-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:07,555 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,45313,1731596105932-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:07,555 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,45313,1731596105932-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:07,555 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-da1c6afcd1f9:45313, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:07,555 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:07,556 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:07,558 DEBUG [master/da1c6afcd1f9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T14:55:07,560 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.372sec 2024-11-14T14:55:07,561 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T14:55:07,561 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T14:55:07,561 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T14:55:07,561 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T14:55:07,561 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T14:55:07,561 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,45313,1731596105932-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:55:07,561 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,45313,1731596105932-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T14:55:07,564 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T14:55:07,564 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T14:55:07,564 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,45313,1731596105932-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:07,574 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a1758ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:55:07,574 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request da1c6afcd1f9,45313,-1 for getting cluster id 2024-11-14T14:55:07,574 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T14:55:07,576 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '20d642ec-f49f-4613-8218-38b819f829d9' 2024-11-14T14:55:07,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T14:55:07,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "20d642ec-f49f-4613-8218-38b819f829d9" 2024-11-14T14:55:07,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33299529, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:55:07,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [da1c6afcd1f9,45313,-1] 2024-11-14T14:55:07,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T14:55:07,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:55:07,579 INFO [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T14:55:07,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27a17641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:55:07,580 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:55:07,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,39339,1731596106071, seqNum=-1] 2024-11-14T14:55:07,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:55:07,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:55:07,585 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=da1c6afcd1f9,45313,1731596105932 2024-11-14T14:55:07,585 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:55:07,588 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-14T14:55:07,588 INFO [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart 2024-11-14T14:55:07,588 INFO [Time-limited test {}] wal.TestLogRolling(323): Replication=2 2024-11-14T14:55:07,589 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T14:55:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is da1c6afcd1f9,45313,1731596105932 2024-11-14T14:55:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@725b07b1 2024-11-14T14:55:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T14:55:07,591 INFO [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T14:55:07,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45313 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-14T14:55:07,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45313 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-14T14:55:07,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45313 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T14:55:07,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45313 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart 2024-11-14T14:55:07,595 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T14:55:07,595 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:07,595 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45313 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnPipelineRestart" procId is: 4 2024-11-14T14:55:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45313 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:55:07,597 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T14:55:07,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42517 is added to blk_1073741835_1011 (size=395) 2024-11-14T14:55:07,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39777 is added to blk_1073741835_1011 (size=395) 2024-11-14T14:55:07,607 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b079a10aee3ff6bb06a1e05ab691b20b, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de 2024-11-14T14:55:07,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42517 is added to blk_1073741836_1012 (size=78) 2024-11-14T14:55:07,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39777 is added to blk_1073741836_1012 (size=78) 2024-11-14T14:55:07,616 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:55:07,616 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing b079a10aee3ff6bb06a1e05ab691b20b, disabling compactions & flushes 2024-11-14T14:55:07,616 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b. 2024-11-14T14:55:07,616 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b. 2024-11-14T14:55:07,616 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b. after waiting 0 ms 2024-11-14T14:55:07,616 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b. 2024-11-14T14:55:07,616 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b. 2024-11-14T14:55:07,616 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for b079a10aee3ff6bb06a1e05ab691b20b: Waiting for close lock at 1731596107616Disabling compacts and flushes for region at 1731596107616Disabling writes for close at 1731596107616Writing region close event to WAL at 1731596107616Closed at 1731596107616 2024-11-14T14:55:07,618 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T14:55:07,618 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731596107618"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731596107618"}]},"ts":"1731596107618"} 2024-11-14T14:55:07,621 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T14:55:07,622 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T14:55:07,623 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731596107622"}]},"ts":"1731596107622"} 2024-11-14T14:55:07,625 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta 2024-11-14T14:55:07,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=b079a10aee3ff6bb06a1e05ab691b20b, ASSIGN}] 2024-11-14T14:55:07,627 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=b079a10aee3ff6bb06a1e05ab691b20b, ASSIGN 2024-11-14T14:55:07,628 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=b079a10aee3ff6bb06a1e05ab691b20b, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,39339,1731596106071; forceNewPlan=false, retain=false 2024-11-14T14:55:07,779 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b079a10aee3ff6bb06a1e05ab691b20b, regionState=OPENING, regionLocation=da1c6afcd1f9,39339,1731596106071 2024-11-14T14:55:07,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=b079a10aee3ff6bb06a1e05ab691b20b, ASSIGN because future has completed 2024-11-14T14:55:07,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b079a10aee3ff6bb06a1e05ab691b20b, server=da1c6afcd1f9,39339,1731596106071}] 2024-11-14T14:55:07,940 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b. 2024-11-14T14:55:07,940 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b079a10aee3ff6bb06a1e05ab691b20b, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:55:07,941 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart b079a10aee3ff6bb06a1e05ab691b20b 2024-11-14T14:55:07,941 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:55:07,941 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b079a10aee3ff6bb06a1e05ab691b20b 2024-11-14T14:55:07,941 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b079a10aee3ff6bb06a1e05ab691b20b 2024-11-14T14:55:07,942 INFO [StoreOpener-b079a10aee3ff6bb06a1e05ab691b20b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region b079a10aee3ff6bb06a1e05ab691b20b 2024-11-14T14:55:07,944 INFO [StoreOpener-b079a10aee3ff6bb06a1e05ab691b20b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b079a10aee3ff6bb06a1e05ab691b20b columnFamilyName info 2024-11-14T14:55:07,944 DEBUG [StoreOpener-b079a10aee3ff6bb06a1e05ab691b20b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:07,944 INFO [StoreOpener-b079a10aee3ff6bb06a1e05ab691b20b-1 {}] regionserver.HStore(327): Store=b079a10aee3ff6bb06a1e05ab691b20b/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:55:07,945 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b079a10aee3ff6bb06a1e05ab691b20b 2024-11-14T14:55:07,945 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/default/TestLogRolling-testLogRollOnPipelineRestart/b079a10aee3ff6bb06a1e05ab691b20b 2024-11-14T14:55:07,946 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/default/TestLogRolling-testLogRollOnPipelineRestart/b079a10aee3ff6bb06a1e05ab691b20b 2024-11-14T14:55:07,946 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b079a10aee3ff6bb06a1e05ab691b20b 2024-11-14T14:55:07,946 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b079a10aee3ff6bb06a1e05ab691b20b 2024-11-14T14:55:07,948 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b079a10aee3ff6bb06a1e05ab691b20b 2024-11-14T14:55:07,951 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/default/TestLogRolling-testLogRollOnPipelineRestart/b079a10aee3ff6bb06a1e05ab691b20b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:55:07,952 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b079a10aee3ff6bb06a1e05ab691b20b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=798687, jitterRate=0.015583887696266174}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:55:07,952 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b079a10aee3ff6bb06a1e05ab691b20b 2024-11-14T14:55:07,953 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b079a10aee3ff6bb06a1e05ab691b20b: Running coprocessor pre-open hook at 1731596107941Writing region info on filesystem at 1731596107941Initializing all the Stores at 1731596107942 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596107942Cleaning up temporary data from old regions at 1731596107946 (+4 ms)Running coprocessor post-open hooks at 1731596107952 (+6 ms)Region opened successfully at 1731596107952 2024-11-14T14:55:07,954 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b., pid=6, masterSystemTime=1731596107936 2024-11-14T14:55:07,956 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b. 2024-11-14T14:55:07,956 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b. 2024-11-14T14:55:07,958 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b079a10aee3ff6bb06a1e05ab691b20b, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,39339,1731596106071 2024-11-14T14:55:07,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b079a10aee3ff6bb06a1e05ab691b20b, server=da1c6afcd1f9,39339,1731596106071 because future has completed 2024-11-14T14:55:07,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T14:55:07,964 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b079a10aee3ff6bb06a1e05ab691b20b, server=da1c6afcd1f9,39339,1731596106071 in 178 msec 2024-11-14T14:55:07,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T14:55:07,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=b079a10aee3ff6bb06a1e05ab691b20b, ASSIGN in 339 msec 2024-11-14T14:55:07,968 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T14:55:07,969 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731596107968"}]},"ts":"1731596107968"} 2024-11-14T14:55:07,971 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta 2024-11-14T14:55:07,972 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T14:55:07,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 380 msec 2024-11-14T14:55:08,512 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:08,521 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:09,512 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:09,522 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:09,616 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T14:55:09,617 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T14:55:09,617 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-11-14T14:55:09,617 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer 2024-11-14T14:55:09,618 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:55:09,618 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T14:55:10,513 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:10,522 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:11,514 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:11,523 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:12,514 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:12,524 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:13,006 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-14T14:55:13,022 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:13,023 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:13,023 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:13,024 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:13,024 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:13,025 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:13,028 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:13,028 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:13,028 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:13,030 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:13,036 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T14:55:13,037 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart' 2024-11-14T14:55:13,515 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:13,525 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:14,516 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:14,525 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:15,516 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:15,526 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:16,517 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:16,527 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:17,518 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:17,527 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:17,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45313 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:55:17,677 INFO [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed 2024-11-14T14:55:17,677 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100 2024-11-14T14:55:17,680 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart 2024-11-14T14:55:17,680 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b. 2024-11-14T14:55:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b., hostname=da1c6afcd1f9,39339,1731596106071, seqNum=2] 2024-11-14T14:55:18,518 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:18,528 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:19,519 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:19,528 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:19,686 INFO [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596107098 2024-11-14T14:55:19,687 WARN [ResponseProcessor for block BP-517382180-172.17.0.2-1731596104998:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-517382180-172.17.0.2-1731596104998:blk_1073741834_1010 java.io.IOException: Bad response ERROR for BP-517382180-172.17.0.2-1731596104998:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:39777,DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:19,687 WARN [DataStreamer for file /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.meta.1731596107496.meta block BP-517382180-172.17.0.2-1731596104998:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-517382180-172.17.0.2-1731596104998:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42517,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK], DatanodeInfoWithStorage[127.0.0.1:39777,DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39777,DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b,DISK]) is bad. 2024-11-14T14:55:19,687 WARN [PacketResponder: BP-517382180-172.17.0.2-1731596104998:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39777] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:19,688 WARN [ResponseProcessor for block BP-517382180-172.17.0.2-1731596104998:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-517382180-172.17.0.2-1731596104998:blk_1073741833_1009 java.io.IOException: Bad response ERROR for BP-517382180-172.17.0.2-1731596104998:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:39777,DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:19,688 WARN [DataStreamer for file /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596107098 block BP-517382180-172.17.0.2-1731596104998:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-517382180-172.17.0.2-1731596104998:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42517,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK], DatanodeInfoWithStorage[127.0.0.1:39777,DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39777,DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b,DISK]) is bad. 2024-11-14T14:55:19,688 WARN [PacketResponder: BP-517382180-172.17.0.2-1731596104998:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39777] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:19,688 WARN [ResponseProcessor for block BP-517382180-172.17.0.2-1731596104998:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-517382180-172.17.0.2-1731596104998:blk_1073741830_1006 java.io.IOException: Bad response ERROR for BP-517382180-172.17.0.2-1731596104998:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:39777,DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:19,688 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1977191407_22 at /127.0.0.1:59000 [Receiving block BP-517382180-172.17.0.2-1731596104998:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:42517:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59000 dst: /127.0.0.1:42517 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:19,688 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1977191407_22 at /127.0.0.1:58988 [Receiving block BP-517382180-172.17.0.2-1731596104998:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:42517:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58988 dst: /127.0.0.1:42517 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:19,688 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1977191407_22 at /127.0.0.1:45570 [Receiving block BP-517382180-172.17.0.2-1731596104998:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:39777:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45570 dst: /127.0.0.1:39777 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:19,689 WARN [DataStreamer for file /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/WALs/da1c6afcd1f9,45313,1731596105932/da1c6afcd1f9%2C45313%2C1731596105932.1731596106675 block BP-517382180-172.17.0.2-1731596104998:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-517382180-172.17.0.2-1731596104998:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:42517,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK], DatanodeInfoWithStorage[127.0.0.1:39777,DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39777,DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b,DISK]) is bad. 2024-11-14T14:55:19,689 WARN [PacketResponder: BP-517382180-172.17.0.2-1731596104998:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39777] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:19,689 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_785841459_22 at /127.0.0.1:58948 [Receiving block BP-517382180-172.17.0.2-1731596104998:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:42517:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58948 dst: /127.0.0.1:42517 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:19,689 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1977191407_22 at /127.0.0.1:45562 [Receiving block BP-517382180-172.17.0.2-1731596104998:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:39777:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45562 dst: /127.0.0.1:39777 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:19,689 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_785841459_22 at /127.0.0.1:45534 [Receiving block BP-517382180-172.17.0.2-1731596104998:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:39777:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45534 dst: /127.0.0.1:39777 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:19,691 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4a7cb65f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:55:19,691 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@49d720be{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:55:19,691 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:55:19,692 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@586e8021{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:55:19,692 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4360f0f4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/hadoop.log.dir/,STOPPED} 2024-11-14T14:55:19,694 WARN [BP-517382180-172.17.0.2-1731596104998 heartbeating to localhost/127.0.0.1:33315 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:55:19,694 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:55:19,694 WARN [BP-517382180-172.17.0.2-1731596104998 heartbeating to localhost/127.0.0.1:33315 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-517382180-172.17.0.2-1731596104998 (Datanode Uuid 1d51cfb9-dd20-4689-a9a3-b54aa021f596) service to localhost/127.0.0.1:33315 2024-11-14T14:55:19,694 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:55:19,694 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/data/data3/current/BP-517382180-172.17.0.2-1731596104998 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:55:19,695 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/data/data4/current/BP-517382180-172.17.0.2-1731596104998 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:55:19,695 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:55:19,709 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:55:19,714 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:55:19,714 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:55:19,714 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:55:19,714 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T14:55:19,715 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@68e23717{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:55:19,715 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@67df601f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:55:19,837 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@71af5be2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/java.io.tmpdir/jetty-localhost-44171-hadoop-hdfs-3_4_1-tests_jar-_-any-2976646110740821025/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:55:19,838 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4a1e644f{HTTP/1.1, (http/1.1)}{localhost:44171} 2024-11-14T14:55:19,838 INFO [Time-limited test {}] server.Server(415): Started @165459ms 2024-11-14T14:55:19,840 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:55:19,860 WARN [ResponseProcessor for block BP-517382180-172.17.0.2-1731596104998:blk_1073741833_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-517382180-172.17.0.2-1731596104998:blk_1073741833_1013 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:19,860 WARN [ResponseProcessor for block BP-517382180-172.17.0.2-1731596104998:blk_1073741834_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-517382180-172.17.0.2-1731596104998:blk_1073741834_1014 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:19,860 WARN [ResponseProcessor for block BP-517382180-172.17.0.2-1731596104998:blk_1073741830_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-517382180-172.17.0.2-1731596104998:blk_1073741830_1015 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:19,861 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1977191407_22 at /127.0.0.1:53782 [Receiving block BP-517382180-172.17.0.2-1731596104998:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:42517:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53782 dst: /127.0.0.1:42517 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:19,861 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1977191407_22 at /127.0.0.1:53786 [Receiving block BP-517382180-172.17.0.2-1731596104998:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:42517:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53786 dst: /127.0.0.1:42517 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:19,861 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_785841459_22 at /127.0.0.1:53792 [Receiving block BP-517382180-172.17.0.2-1731596104998:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:42517:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53792 dst: /127.0.0.1:42517 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:19,867 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@45628471{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:55:19,868 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4e6129e0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:55:19,868 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:55:19,868 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@33255ae1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:55:19,868 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74ea1d44{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/hadoop.log.dir/,STOPPED} 2024-11-14T14:55:19,870 WARN [BP-517382180-172.17.0.2-1731596104998 heartbeating to localhost/127.0.0.1:33315 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:55:19,870 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:55:19,870 WARN [BP-517382180-172.17.0.2-1731596104998 heartbeating to localhost/127.0.0.1:33315 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-517382180-172.17.0.2-1731596104998 (Datanode Uuid 972bbbb2-749b-46dd-bc90-7ff3b222239e) service to localhost/127.0.0.1:33315 2024-11-14T14:55:19,870 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:55:19,870 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/data/data1/current/BP-517382180-172.17.0.2-1731596104998 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:55:19,870 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/data/data2/current/BP-517382180-172.17.0.2-1731596104998 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:55:19,871 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:55:19,879 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:55:19,882 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:55:19,883 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:55:19,883 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:55:19,883 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:55:19,884 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31267d1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:55:19,884 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b9f4dad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:55:19,969 WARN [Thread-1349 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:55:19,972 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb9394d42b5393ec with lease ID 0x67fbf7156c0a9284: from storage DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b node DatanodeRegistration(127.0.0.1:33105, datanodeUuid=1d51cfb9-dd20-4689-a9a3-b54aa021f596, infoPort=43007, infoSecurePort=0, ipcPort=37909, storageInfo=lv=-57;cid=testClusterID;nsid=1672118275;c=1731596104998), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:55:19,972 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb9394d42b5393ec with lease ID 0x67fbf7156c0a9284: from storage DS-9f179d08-6877-4364-bb7b-916f3d937328 node DatanodeRegistration(127.0.0.1:33105, datanodeUuid=1d51cfb9-dd20-4689-a9a3-b54aa021f596, infoPort=43007, infoSecurePort=0, ipcPort=37909, storageInfo=lv=-57;cid=testClusterID;nsid=1672118275;c=1731596104998), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:55:20,035 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@48dc7e0c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/java.io.tmpdir/jetty-localhost-37757-hadoop-hdfs-3_4_1-tests_jar-_-any-15681773273880232740/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:55:20,036 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6beed7e8{HTTP/1.1, (http/1.1)}{localhost:37757} 2024-11-14T14:55:20,036 INFO [Time-limited test {}] server.Server(415): Started @165658ms 2024-11-14T14:55:20,037 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:55:20,135 WARN [Thread-1380 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:55:20,138 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x105a8d1a09fa6d6b with lease ID 0x67fbf7156c0a9285: from storage DS-42e12750-ea13-4781-9ab0-d5facb94aa61 node DatanodeRegistration(127.0.0.1:39237, datanodeUuid=972bbbb2-749b-46dd-bc90-7ff3b222239e, infoPort=46555, infoSecurePort=0, ipcPort=42993, storageInfo=lv=-57;cid=testClusterID;nsid=1672118275;c=1731596104998), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:55:20,138 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x105a8d1a09fa6d6b with lease ID 0x67fbf7156c0a9285: from storage DS-790acd79-c81c-4653-b697-859334b8bfee node DatanodeRegistration(127.0.0.1:39237, datanodeUuid=972bbbb2-749b-46dd-bc90-7ff3b222239e, infoPort=46555, infoSecurePort=0, ipcPort=42993, storageInfo=lv=-57;cid=testClusterID;nsid=1672118275;c=1731596104998), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:55:20,520 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:20,529 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:21,066 INFO [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted 2024-11-14T14:55:21,069 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002 2024-11-14T14:55:21,071 ERROR [FSHLog-0-hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de-prefix:da1c6afcd1f9,39339,1731596106071 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42517,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:21,071 WARN [FSHLog-0-hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de-prefix:da1c6afcd1f9,39339,1731596106071 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42517,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:21,072 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog da1c6afcd1f9%2C39339%2C1731596106071:(num 1731596107098) roll requested 2024-11-14T14:55:21,072 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C39339%2C1731596106071.1731596121072 2024-11-14T14:55:21,082 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596107098 newFile=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596121072 2024-11-14T14:55:21,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:21,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:21,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:21,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:21,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:21,083 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596107098 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596121072 2024-11-14T14:55:21,088 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42517,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:21,088 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42517,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:21,088 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596107098 2024-11-14T14:55:21,089 WARN [IPC Server handler 0 on default port 33315 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596107098 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1013 2024-11-14T14:55:21,089 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596107098 after 1ms 2024-11-14T14:55:21,092 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43007:43007),(127.0.0.1/127.0.0.1:46555:46555)] 2024-11-14T14:55:21,092 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596107098 is not closed yet, will try archiving it next time 2024-11-14T14:55:21,520 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:21,529 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:22,521 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:22,530 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:23,096 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003 2024-11-14T14:55:23,522 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:23,531 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:23,972 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741833_1013: GenerationStamp not matched, existing replica is blk_1073741833_1009 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-14T14:55:24,522 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:24,531 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:25,090 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596107098 after 4002ms 2024-11-14T14:55:25,100 WARN [ResponseProcessor for block BP-517382180-172.17.0.2-1731596104998:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-517382180-172.17.0.2-1731596104998:blk_1073741837_1016 java.io.IOException: Bad response ERROR for BP-517382180-172.17.0.2-1731596104998:blk_1073741837_1016 from datanode DatanodeInfoWithStorage[127.0.0.1:39237,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:25,100 WARN [DataStreamer for file /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596121072 block BP-517382180-172.17.0.2-1731596104998:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-517382180-172.17.0.2-1731596104998:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33105,DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b,DISK], DatanodeInfoWithStorage[127.0.0.1:39237,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39237,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK]) is bad. 2024-11-14T14:55:25,100 WARN [PacketResponder: BP-517382180-172.17.0.2-1731596104998:blk_1073741837_1016, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:39237] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:25,100 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1977191407_22 at /127.0.0.1:37796 [Receiving block BP-517382180-172.17.0.2-1731596104998:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:33105:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37796 dst: /127.0.0.1:33105 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:25,101 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1977191407_22 at /127.0.0.1:59086 [Receiving block BP-517382180-172.17.0.2-1731596104998:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:39237:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59086 dst: /127.0.0.1:39237 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:25,102 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@48dc7e0c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:55:25,102 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6beed7e8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:55:25,102 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:55:25,102 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5b9f4dad{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:55:25,102 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31267d1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/hadoop.log.dir/,STOPPED} 2024-11-14T14:55:25,104 WARN [BP-517382180-172.17.0.2-1731596104998 heartbeating to localhost/127.0.0.1:33315 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:55:25,104 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:55:25,104 WARN [BP-517382180-172.17.0.2-1731596104998 heartbeating to localhost/127.0.0.1:33315 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-517382180-172.17.0.2-1731596104998 (Datanode Uuid 972bbbb2-749b-46dd-bc90-7ff3b222239e) service to localhost/127.0.0.1:33315 2024-11-14T14:55:25,104 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:55:25,104 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/data/data1/current/BP-517382180-172.17.0.2-1731596104998 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:55:25,105 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/data/data2/current/BP-517382180-172.17.0.2-1731596104998 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:55:25,105 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:55:25,114 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:55:25,117 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:55:25,119 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:55:25,119 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:55:25,119 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:55:25,119 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20432799{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:55:25,120 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2378632b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:55:25,260 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4eef3a93{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/java.io.tmpdir/jetty-localhost-38347-hadoop-hdfs-3_4_1-tests_jar-_-any-14837563206101196381/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:55:25,261 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d8da7be{HTTP/1.1, (http/1.1)}{localhost:38347} 2024-11-14T14:55:25,261 INFO [Time-limited test {}] server.Server(415): Started @170883ms 2024-11-14T14:55:25,263 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:55:25,317 WARN [ResponseProcessor for block BP-517382180-172.17.0.2-1731596104998:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-517382180-172.17.0.2-1731596104998:blk_1073741837_1018 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:25,318 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1977191407_22 at /127.0.0.1:37820 [Receiving block BP-517382180-172.17.0.2-1731596104998:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:33105:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37820 dst: /127.0.0.1:33105 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:25,342 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@71af5be2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:55:25,343 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4a1e644f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:55:25,343 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:55:25,343 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@67df601f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:55:25,343 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@68e23717{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/hadoop.log.dir/,STOPPED} 2024-11-14T14:55:25,346 WARN [BP-517382180-172.17.0.2-1731596104998 heartbeating to localhost/127.0.0.1:33315 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:55:25,346 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:55:25,346 WARN [BP-517382180-172.17.0.2-1731596104998 heartbeating to localhost/127.0.0.1:33315 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-517382180-172.17.0.2-1731596104998 (Datanode Uuid 1d51cfb9-dd20-4689-a9a3-b54aa021f596) service to localhost/127.0.0.1:33315 2024-11-14T14:55:25,346 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:55:25,347 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/data/data3/current/BP-517382180-172.17.0.2-1731596104998 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:55:25,347 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/data/data4/current/BP-517382180-172.17.0.2-1731596104998 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:55:25,347 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:55:25,367 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:55:25,371 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:55:25,376 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:55:25,376 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:55:25,377 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T14:55:25,381 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1decdda3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:55:25,381 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4251f41a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:55:25,420 WARN [Thread-1423 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:55:25,427 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9056d245b07c3752 with lease ID 0x67fbf7156c0a9286: from storage DS-42e12750-ea13-4781-9ab0-d5facb94aa61 node DatanodeRegistration(127.0.0.1:44325, datanodeUuid=972bbbb2-749b-46dd-bc90-7ff3b222239e, infoPort=45187, infoSecurePort=0, ipcPort=39323, storageInfo=lv=-57;cid=testClusterID;nsid=1672118275;c=1731596104998), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:55:25,427 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9056d245b07c3752 with lease ID 0x67fbf7156c0a9286: from storage DS-790acd79-c81c-4653-b697-859334b8bfee node DatanodeRegistration(127.0.0.1:44325, datanodeUuid=972bbbb2-749b-46dd-bc90-7ff3b222239e, infoPort=45187, infoSecurePort=0, ipcPort=39323, storageInfo=lv=-57;cid=testClusterID;nsid=1672118275;c=1731596104998), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:55:25,523 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:25,530 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@22586a60{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/java.io.tmpdir/jetty-localhost-34567-hadoop-hdfs-3_4_1-tests_jar-_-any-6179351670840130869/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:55:25,530 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5684bce2{HTTP/1.1, (http/1.1)}{localhost:34567} 2024-11-14T14:55:25,530 INFO [Time-limited test {}] server.Server(415): Started @171152ms 2024-11-14T14:55:25,532 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:25,532 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:55:25,629 WARN [Thread-1454 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:55:25,631 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x90e8f1db100d471 with lease ID 0x67fbf7156c0a9287: from storage DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b node DatanodeRegistration(127.0.0.1:41267, datanodeUuid=1d51cfb9-dd20-4689-a9a3-b54aa021f596, infoPort=40621, infoSecurePort=0, ipcPort=38443, storageInfo=lv=-57;cid=testClusterID;nsid=1672118275;c=1731596104998), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:55:25,632 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x90e8f1db100d471 with lease ID 0x67fbf7156c0a9287: from storage DS-9f179d08-6877-4364-bb7b-916f3d937328 node DatanodeRegistration(127.0.0.1:41267, datanodeUuid=1d51cfb9-dd20-4689-a9a3-b54aa021f596, infoPort=40621, infoSecurePort=0, ipcPort=38443, storageInfo=lv=-57;cid=testClusterID;nsid=1672118275;c=1731596104998), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:55:26,524 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:26,532 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:26,554 INFO [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted 2024-11-14T14:55:26,556 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004 2024-11-14T14:55:26,557 ERROR [FSHLog-0-hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de-prefix:da1c6afcd1f9,39339,1731596106071 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33105,DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:26,557 WARN [FSHLog-0-hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de-prefix:da1c6afcd1f9,39339,1731596106071 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33105,DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:26,558 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog da1c6afcd1f9%2C39339%2C1731596106071:(num 1731596121072) roll requested 2024-11-14T14:55:26,558 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C39339%2C1731596106071.1731596126558 2024-11-14T14:55:26,567 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596121072 newFile=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596126558 2024-11-14T14:55:26,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:26,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:26,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:26,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:26,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:26,568 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596121072 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596126558 2024-11-14T14:55:26,568 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33105,DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:26,569 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33105,DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:26,569 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596121072 2024-11-14T14:55:26,569 WARN [IPC Server handler 4 on default port 33315 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596121072 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018 2024-11-14T14:55:26,569 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596121072 after 0ms 2024-11-14T14:55:26,577 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40621:40621),(127.0.0.1/127.0.0.1:45187:45187)] 2024-11-14T14:55:26,577 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596121072 is not closed yet, will try archiving it next time 2024-11-14T14:55:27,524 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:27,533 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:28,525 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:28,533 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:28,579 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C39339%2C1731596106071.1731596128579 2024-11-14T14:55:28,585 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596126558 newFile=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596128579 2024-11-14T14:55:28,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:28,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:28,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:28,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:28,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:28,586 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596126558 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596128579 2024-11-14T14:55:28,587 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45187:45187),(127.0.0.1/127.0.0.1:40621:40621)] 2024-11-14T14:55:28,587 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596121072 is not closed yet, will try archiving it next time 2024-11-14T14:55:28,587 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596126558 is not closed yet, will try archiving it next time 2024-11-14T14:55:28,587 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596107098 2024-11-14T14:55:28,587 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596107098 2024-11-14T14:55:28,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44325 is added to blk_1073741838_1019 (size=1264) 2024-11-14T14:55:28,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41267 is added to blk_1073741838_1019 (size=1264) 2024-11-14T14:55:28,588 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596107098 after 1ms 2024-11-14T14:55:28,588 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596107098 2024-11-14T14:55:28,588 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596121072 is not closed yet, will try archiving it next time 2024-11-14T14:55:28,598 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1731596107953/Put/vlen=218/seqid=0] 2024-11-14T14:55:28,598 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1731596117684/Put/vlen=1045/seqid=0] 2024-11-14T14:55:28,598 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596107098 2024-11-14T14:55:28,598 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596121072 2024-11-14T14:55:28,598 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596121072 2024-11-14T14:55:28,599 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596121072 after 0ms 2024-11-14T14:55:28,599 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596121072 2024-11-14T14:55:28,603 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1731596121071/Put/vlen=1045/seqid=0] 2024-11-14T14:55:28,603 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1731596123097/Put/vlen=1045/seqid=0] 2024-11-14T14:55:28,603 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596121072 2024-11-14T14:55:28,603 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596126558 2024-11-14T14:55:28,603 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596126558 2024-11-14T14:55:28,604 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596126558 after 1ms 2024-11-14T14:55:28,604 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596126558 2024-11-14T14:55:28,607 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1731596126557/Put/vlen=1045/seqid=0] 2024-11-14T14:55:28,607 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596128579 2024-11-14T14:55:28,608 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596128579 2024-11-14T14:55:28,608 WARN [IPC Server handler 1 on default port 33315 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596128579 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021 2024-11-14T14:55:28,608 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596128579 after 0ms 2024-11-14T14:55:29,428 WARN [ResponseProcessor for block BP-517382180-172.17.0.2-1731596104998:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-517382180-172.17.0.2-1731596104998:blk_1073741839_1021 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:29,428 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_785841459_22 at /127.0.0.1:49126 [Receiving block BP-517382180-172.17.0.2-1731596104998:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:44325:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49126 dst: /127.0.0.1:44325 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44325 remote=/127.0.0.1:49126]. Total timeout mills is 60000, 59157 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:29,428 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_785841459_22 at /127.0.0.1:47806 [Receiving block BP-517382180-172.17.0.2-1731596104998:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:41267:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47806 dst: /127.0.0.1:41267 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:55:29,428 WARN [DataStreamer for file /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596128579 block BP-517382180-172.17.0.2-1731596104998:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-517382180-172.17.0.2-1731596104998:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44325,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK], DatanodeInfoWithStorage[127.0.0.1:41267,DS-fe9a35b0-a539-448d-adab-1ff91a62fe6b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44325,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK]) is bad. 2024-11-14T14:55:29,429 WARN [DataStreamer for file /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596128579 block BP-517382180-172.17.0.2-1731596104998:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-517382180-172.17.0.2-1731596104998:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:29,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44325 is added to blk_1073741839_1022 (size=85) 2024-11-14T14:55:29,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41267 is added to blk_1073741839_1022 (size=85) 2024-11-14T14:55:29,526 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:29,534 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:30,427 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-14T14:55:30,526 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:30,535 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:30,570 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596121072 after 4001ms 2024-11-14T14:55:31,527 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:31,535 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:32,528 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:32,536 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:32,609 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596128579 after 4001ms 2024-11-14T14:55:32,609 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596128579 2024-11-14T14:55:32,613 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596128579 2024-11-14T14:55:32,614 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB 2024-11-14T14:55:32,614 ERROR [FSHLog-0-hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de-prefix:da1c6afcd1f9,39339,1731596106071.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42517,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:32,614 WARN [FSHLog-0-hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de-prefix:da1c6afcd1f9,39339,1731596106071.meta {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42517,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:32,614 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog da1c6afcd1f9%2C39339%2C1731596106071.meta:.meta(num 1731596107496) roll requested 2024-11-14T14:55:32,615 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C39339%2C1731596106071.meta.1731596132614.meta 2024-11-14T14:55:32,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:32,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:32,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:32,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:32,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:32,621 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.meta.1731596107496.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.meta.1731596132614.meta 2024-11-14T14:55:32,622 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42517,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:32,622 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42517,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:32,622 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.meta.1731596107496.meta 2024-11-14T14:55:32,622 WARN [IPC Server handler 0 on default port 33315 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.meta.1731596107496.meta has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741834_1014 2024-11-14T14:55:32,623 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.meta.1731596107496.meta after 1ms 2024-11-14T14:55:32,628 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45187:45187),(127.0.0.1/127.0.0.1:40621:40621)] 2024-11-14T14:55:32,628 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.meta.1731596107496.meta is not closed yet, will try archiving it next time 2024-11-14T14:55:32,649 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740/.tmp/info/97321f16af6548f680a829aca52f7faf is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b./info:regioninfo/1731596107957/Put/seqid=0 2024-11-14T14:55:32,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41267 is added to blk_1073741841_1025 (size=7125) 2024-11-14T14:55:32,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44325 is added to blk_1073741841_1025 (size=7125) 2024-11-14T14:55:32,655 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740/.tmp/info/97321f16af6548f680a829aca52f7faf 2024-11-14T14:55:32,682 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740/.tmp/ns/b92e7dafa2ee47a49101da8707ba7f30 is 43, key is default/ns:d/1731596107545/Put/seqid=0 2024-11-14T14:55:32,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41267 is added to blk_1073741842_1026 (size=5153) 2024-11-14T14:55:32,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44325 is added to blk_1073741842_1026 (size=5153) 2024-11-14T14:55:32,695 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740/.tmp/ns/b92e7dafa2ee47a49101da8707ba7f30 2024-11-14T14:55:32,716 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740/.tmp/table/78bbda3812c3409e8ad028b13242c9b8 is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1731596107968/Put/seqid=0 2024-11-14T14:55:32,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41267 is added to blk_1073741843_1027 (size=5438) 2024-11-14T14:55:32,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44325 is added to blk_1073741843_1027 (size=5438) 2024-11-14T14:55:32,723 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740/.tmp/table/78bbda3812c3409e8ad028b13242c9b8 2024-11-14T14:55:32,729 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740/.tmp/info/97321f16af6548f680a829aca52f7faf as hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740/info/97321f16af6548f680a829aca52f7faf 2024-11-14T14:55:32,735 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740/info/97321f16af6548f680a829aca52f7faf, entries=10, sequenceid=11, filesize=7.0 K 2024-11-14T14:55:32,736 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740/.tmp/ns/b92e7dafa2ee47a49101da8707ba7f30 as hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740/ns/b92e7dafa2ee47a49101da8707ba7f30 2024-11-14T14:55:32,742 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740/ns/b92e7dafa2ee47a49101da8707ba7f30, entries=2, sequenceid=11, filesize=5.0 K 2024-11-14T14:55:32,743 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740/.tmp/table/78bbda3812c3409e8ad028b13242c9b8 as hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740/table/78bbda3812c3409e8ad028b13242c9b8 2024-11-14T14:55:32,749 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740/table/78bbda3812c3409e8ad028b13242c9b8, entries=2, sequenceid=11, filesize=5.3 K 2024-11-14T14:55:32,751 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 137ms, sequenceid=11, compaction requested=false 2024-11-14T14:55:32,751 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-14T14:55:32,751 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b079a10aee3ff6bb06a1e05ab691b20b 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB 2024-11-14T14:55:32,751 ERROR [FSHLog-0-hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de-prefix:da1c6afcd1f9,39339,1731596106071 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-517382180-172.17.0.2-1731596104998:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:32,752 WARN [FSHLog-0-hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de-prefix:da1c6afcd1f9,39339,1731596106071 {}] wal.AbstractFSWAL(2174): append entry failed org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-517382180-172.17.0.2-1731596104998:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:32,752 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog da1c6afcd1f9%2C39339%2C1731596106071:(num 1731596128579) roll requested 2024-11-14T14:55:32,752 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C39339%2C1731596106071.1731596132752 2024-11-14T14:55:32,758 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596128579 newFile=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596132752 2024-11-14T14:55:32,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:32,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:32,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:32,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:32,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:32,759 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596128579 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596132752 2024-11-14T14:55:32,759 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-517382180-172.17.0.2-1731596104998:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:32,759 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-517382180-172.17.0.2-1731596104998:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:32,759 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596128579 2024-11-14T14:55:32,760 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596128579 after 1ms 2024-11-14T14:55:32,764 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.1731596128579 to hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/oldWALs/da1c6afcd1f9%2C39339%2C1731596106071.1731596128579 2024-11-14T14:55:32,772 DEBUG [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45187:45187),(127.0.0.1/127.0.0.1:40621:40621)] 2024-11-14T14:55:32,791 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/default/TestLogRolling-testLogRollOnPipelineRestart/b079a10aee3ff6bb06a1e05ab691b20b/.tmp/info/3b3b678c47a042aab8e1327a4671ea7a is 1080, key is row1002/info:/1731596117684/Put/seqid=0 2024-11-14T14:55:32,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41267 is added to blk_1073741845_1029 (size=9270) 2024-11-14T14:55:32,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44325 is added to blk_1073741845_1029 (size=9270) 2024-11-14T14:55:32,797 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/default/TestLogRolling-testLogRollOnPipelineRestart/b079a10aee3ff6bb06a1e05ab691b20b/.tmp/info/3b3b678c47a042aab8e1327a4671ea7a 2024-11-14T14:55:32,805 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/default/TestLogRolling-testLogRollOnPipelineRestart/b079a10aee3ff6bb06a1e05ab691b20b/.tmp/info/3b3b678c47a042aab8e1327a4671ea7a as hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/default/TestLogRolling-testLogRollOnPipelineRestart/b079a10aee3ff6bb06a1e05ab691b20b/info/3b3b678c47a042aab8e1327a4671ea7a 2024-11-14T14:55:32,811 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/default/TestLogRolling-testLogRollOnPipelineRestart/b079a10aee3ff6bb06a1e05ab691b20b/info/3b3b678c47a042aab8e1327a4671ea7a, entries=4, sequenceid=8, filesize=9.1 K 2024-11-14T14:55:32,813 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for b079a10aee3ff6bb06a1e05ab691b20b in 62ms, sequenceid=8, compaction requested=false 2024-11-14T14:55:32,813 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b079a10aee3ff6bb06a1e05ab691b20b: 2024-11-14T14:55:32,819 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T14:55:32,819 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:55:32,819 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:55:32,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:55:32,819 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T14:55:32,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:55:32,820 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T14:55:32,820 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1330077295, stopped=false 2024-11-14T14:55:32,820 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=da1c6afcd1f9,45313,1731596105932 2024-11-14T14:55:32,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:55:32,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:55:32,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:32,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:32,822 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:55:32,822 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:55:32,822 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:55:32,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:55:32,823 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'da1c6afcd1f9,39339,1731596106071' ***** 2024-11-14T14:55:32,823 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T14:55:32,823 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:55:32,823 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:55:32,823 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T14:55:32,823 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T14:55:32,823 INFO [RS:0;da1c6afcd1f9:39339 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T14:55:32,823 INFO [RS:0;da1c6afcd1f9:39339 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T14:55:32,823 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.HRegionServer(3091): Received CLOSE for b079a10aee3ff6bb06a1e05ab691b20b 2024-11-14T14:55:32,824 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.HRegionServer(959): stopping server da1c6afcd1f9,39339,1731596106071 2024-11-14T14:55:32,824 INFO [RS:0;da1c6afcd1f9:39339 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:55:32,824 INFO [RS:0;da1c6afcd1f9:39339 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;da1c6afcd1f9:39339. 2024-11-14T14:55:32,824 DEBUG [RS:0;da1c6afcd1f9:39339 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:55:32,824 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b079a10aee3ff6bb06a1e05ab691b20b, disabling compactions & flushes 2024-11-14T14:55:32,824 DEBUG [RS:0;da1c6afcd1f9:39339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:55:32,824 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b. 2024-11-14T14:55:32,824 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b. 2024-11-14T14:55:32,824 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T14:55:32,824 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T14:55:32,824 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b. after waiting 0 ms 2024-11-14T14:55:32,824 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T14:55:32,824 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b. 2024-11-14T14:55:32,824 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T14:55:32,824 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-14T14:55:32,825 DEBUG [RS:0;da1c6afcd1f9:39339 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, b079a10aee3ff6bb06a1e05ab691b20b=TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b.} 2024-11-14T14:55:32,825 DEBUG [RS:0;da1c6afcd1f9:39339 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, b079a10aee3ff6bb06a1e05ab691b20b 2024-11-14T14:55:32,825 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:55:32,825 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:55:32,825 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:55:32,825 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:55:32,825 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:55:32,833 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/default/TestLogRolling-testLogRollOnPipelineRestart/b079a10aee3ff6bb06a1e05ab691b20b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-14T14:55:32,833 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b. 2024-11-14T14:55:32,834 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b079a10aee3ff6bb06a1e05ab691b20b: Waiting for close lock at 1731596132824Running coprocessor pre-close hooks at 1731596132824Disabling compacts and flushes for region at 1731596132824Disabling writes for close at 1731596132824Writing region close event to WAL at 1731596132828 (+4 ms)Running coprocessor post-close hooks at 1731596132833 (+5 ms)Closed at 1731596132833 2024-11-14T14:55:32,834 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731596107592.b079a10aee3ff6bb06a1e05ab691b20b. 2024-11-14T14:55:32,834 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-14T14:55:32,834 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:55:32,834 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:55:32,834 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731596132825Running coprocessor pre-close hooks at 1731596132825Disabling compacts and flushes for region at 1731596132825Disabling writes for close at 1731596132825Writing region close event to WAL at 1731596132829 (+4 ms)Running coprocessor post-close hooks at 1731596132834 (+5 ms)Closed at 1731596132834 2024-11-14T14:55:32,835 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T14:55:32,961 INFO [regionserver/da1c6afcd1f9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:55:32,986 INFO [regionserver/da1c6afcd1f9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T14:55:32,986 INFO [regionserver/da1c6afcd1f9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T14:55:33,025 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.HRegionServer(976): stopping server da1c6afcd1f9,39339,1731596106071; all regions closed. 2024-11-14T14:55:33,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:33,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:33,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:33,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:33,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:33,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41267 is added to blk_1073741840_1023 (size=825) 2024-11-14T14:55:33,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44325 is added to blk_1073741840_1023 (size=825) 2024-11-14T14:55:33,528 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:33,536 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:34,529 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:34,537 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:35,529 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:35,537 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:35,631 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741834_1014: GenerationStamp not matched, existing replica is blk_1073741834_1010 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-14T14:55:35,888 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T14:55:36,530 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:36,538 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:36,624 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.meta.1731596107496.meta after 4001ms 2024-11-14T14:55:36,624 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/WALs/da1c6afcd1f9,39339,1731596106071/da1c6afcd1f9%2C39339%2C1731596106071.meta.1731596107496.meta to hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/oldWALs/da1c6afcd1f9%2C39339%2C1731596106071.meta.1731596107496.meta 2024-11-14T14:55:36,627 DEBUG [RS:0;da1c6afcd1f9:39339 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/oldWALs 2024-11-14T14:55:36,627 INFO [RS:0;da1c6afcd1f9:39339 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog da1c6afcd1f9%2C39339%2C1731596106071.meta:.meta(num 1731596132614) 2024-11-14T14:55:36,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:36,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:36,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:36,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:36,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:36,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41267 is added to blk_1073741844_1028 (size=1162) 2024-11-14T14:55:36,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44325 is added to blk_1073741844_1028 (size=1162) 2024-11-14T14:55:36,636 DEBUG [RS:0;da1c6afcd1f9:39339 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/oldWALs 2024-11-14T14:55:36,636 INFO [RS:0;da1c6afcd1f9:39339 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog da1c6afcd1f9%2C39339%2C1731596106071:(num 1731596132752) 2024-11-14T14:55:36,636 DEBUG [RS:0;da1c6afcd1f9:39339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:55:36,636 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:55:36,636 INFO [RS:0;da1c6afcd1f9:39339 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:55:36,637 INFO [RS:0;da1c6afcd1f9:39339 {}] hbase.ChoreService(370): Chore service for: regionserver/da1c6afcd1f9:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T14:55:36,637 INFO [RS:0;da1c6afcd1f9:39339 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:55:36,637 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:55:36,637 INFO [RS:0;da1c6afcd1f9:39339 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39339 2024-11-14T14:55:36,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/da1c6afcd1f9,39339,1731596106071 2024-11-14T14:55:36,639 INFO [RS:0;da1c6afcd1f9:39339 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:55:36,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:55:36,641 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [da1c6afcd1f9,39339,1731596106071] 2024-11-14T14:55:36,642 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/da1c6afcd1f9,39339,1731596106071 already deleted, retry=false 2024-11-14T14:55:36,642 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; da1c6afcd1f9,39339,1731596106071 expired; onlineServers=0 2024-11-14T14:55:36,642 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'da1c6afcd1f9,45313,1731596105932' ***** 2024-11-14T14:55:36,642 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T14:55:36,642 INFO [M:0;da1c6afcd1f9:45313 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:55:36,642 INFO [M:0;da1c6afcd1f9:45313 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:55:36,642 DEBUG [M:0;da1c6afcd1f9:45313 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T14:55:36,643 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T14:55:36,643 DEBUG [M:0;da1c6afcd1f9:45313 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T14:55:36,643 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731596106848 {}] cleaner.HFileCleaner(306): Exit Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731596106848,5,FailOnTimeoutGroup] 2024-11-14T14:55:36,643 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731596106840 {}] cleaner.HFileCleaner(306): Exit Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731596106840,5,FailOnTimeoutGroup] 2024-11-14T14:55:36,643 INFO [M:0;da1c6afcd1f9:45313 {}] hbase.ChoreService(370): Chore service for: master/da1c6afcd1f9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T14:55:36,643 INFO [M:0;da1c6afcd1f9:45313 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:55:36,643 DEBUG [M:0;da1c6afcd1f9:45313 {}] master.HMaster(1795): Stopping service threads 2024-11-14T14:55:36,643 INFO [M:0;da1c6afcd1f9:45313 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T14:55:36,643 INFO [M:0;da1c6afcd1f9:45313 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:55:36,644 INFO [M:0;da1c6afcd1f9:45313 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:55:36,644 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T14:55:36,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T14:55:36,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:36,644 DEBUG [M:0;da1c6afcd1f9:45313 {}] zookeeper.ZKUtil(347): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T14:55:36,645 WARN [M:0;da1c6afcd1f9:45313 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T14:55:36,645 INFO [M:0;da1c6afcd1f9:45313 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/.lastflushedseqids 2024-11-14T14:55:36,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44325 is added to blk_1073741846_1030 (size=120) 2024-11-14T14:55:36,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41267 is added to blk_1073741846_1030 (size=120) 2024-11-14T14:55:36,652 INFO [M:0;da1c6afcd1f9:45313 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:55:36,652 INFO [M:0;da1c6afcd1f9:45313 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T14:55:36,653 DEBUG [M:0;da1c6afcd1f9:45313 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:55:36,653 INFO [M:0;da1c6afcd1f9:45313 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:55:36,653 DEBUG [M:0;da1c6afcd1f9:45313 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:55:36,653 DEBUG [M:0;da1c6afcd1f9:45313 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:55:36,653 DEBUG [M:0;da1c6afcd1f9:45313 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:55:36,653 INFO [M:0;da1c6afcd1f9:45313 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB 2024-11-14T14:55:36,653 ERROR [FSHLog-0-hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData-prefix:da1c6afcd1f9,45313,1731596105932 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42517,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:36,653 WARN [FSHLog-0-hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData-prefix:da1c6afcd1f9,45313,1731596105932 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42517,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:36,653 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog da1c6afcd1f9%2C45313%2C1731596105932:(num 1731596106675) roll requested 2024-11-14T14:55:36,654 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C45313%2C1731596105932.1731596136654 2024-11-14T14:55:36,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:36,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:36,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:36,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:36,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:36,660 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/WALs/da1c6afcd1f9,45313,1731596105932/da1c6afcd1f9%2C45313%2C1731596105932.1731596106675 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/WALs/da1c6afcd1f9,45313,1731596105932/da1c6afcd1f9%2C45313%2C1731596105932.1731596136654 2024-11-14T14:55:36,660 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42517,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:36,660 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:42517,DS-42e12750-ea13-4781-9ab0-d5facb94aa61,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-14T14:55:36,660 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/WALs/da1c6afcd1f9,45313,1731596105932/da1c6afcd1f9%2C45313%2C1731596105932.1731596106675 2024-11-14T14:55:36,661 WARN [IPC Server handler 4 on default port 33315 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/WALs/da1c6afcd1f9,45313,1731596105932/da1c6afcd1f9%2C45313%2C1731596105932.1731596106675 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1015 2024-11-14T14:55:36,661 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/WALs/da1c6afcd1f9,45313,1731596105932/da1c6afcd1f9%2C45313%2C1731596105932.1731596106675 after 1ms 2024-11-14T14:55:36,664 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40621:40621),(127.0.0.1/127.0.0.1:45187:45187)] 2024-11-14T14:55:36,664 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/WALs/da1c6afcd1f9,45313,1731596105932/da1c6afcd1f9%2C45313%2C1731596105932.1731596106675 is not closed yet, will try archiving it next time 2024-11-14T14:55:36,683 DEBUG [M:0;da1c6afcd1f9:45313 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ddc70494fc5e4272bd980d468cbe4539 is 82, key is hbase:meta,,1/info:regioninfo/1731596107525/Put/seqid=0 2024-11-14T14:55:36,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44325 is added to blk_1073741848_1033 (size=5672) 2024-11-14T14:55:36,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41267 is added to blk_1073741848_1033 (size=5672) 2024-11-14T14:55:36,703 INFO [M:0;da1c6afcd1f9:45313 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ddc70494fc5e4272bd980d468cbe4539 2024-11-14T14:55:36,727 DEBUG [M:0;da1c6afcd1f9:45313 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3a21f0758f544d0681b86f04ee18f45c is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731596107973/Put/seqid=0 2024-11-14T14:55:36,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41267 is added to blk_1073741849_1034 (size=6118) 2024-11-14T14:55:36,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44325 is added to blk_1073741849_1034 (size=6118) 2024-11-14T14:55:36,735 INFO [M:0;da1c6afcd1f9:45313 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3a21f0758f544d0681b86f04ee18f45c 2024-11-14T14:55:36,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:55:36,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39339-0x101a7472e0b0001, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:55:36,741 INFO [RS:0;da1c6afcd1f9:39339 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:55:36,741 INFO [RS:0;da1c6afcd1f9:39339 {}] regionserver.HRegionServer(1031): Exiting; stopping=da1c6afcd1f9,39339,1731596106071; zookeeper connection closed. 2024-11-14T14:55:36,741 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@54e3af72 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@54e3af72 2024-11-14T14:55:36,741 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T14:55:36,756 DEBUG [M:0;da1c6afcd1f9:45313 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2a7b367d2af5407e9473a3dc689bcb3b is 69, key is da1c6afcd1f9,39339,1731596106071/rs:state/1731596106910/Put/seqid=0 2024-11-14T14:55:36,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41267 is added to blk_1073741850_1035 (size=5156) 2024-11-14T14:55:36,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44325 is added to blk_1073741850_1035 (size=5156) 2024-11-14T14:55:36,761 INFO [M:0;da1c6afcd1f9:45313 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2a7b367d2af5407e9473a3dc689bcb3b 2024-11-14T14:55:36,782 DEBUG [M:0;da1c6afcd1f9:45313 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ec2f7adb7982446b91adeec27827ac3a is 52, key is load_balancer_on/state:d/1731596107587/Put/seqid=0 2024-11-14T14:55:36,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41267 is added to blk_1073741851_1036 (size=5056) 2024-11-14T14:55:36,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44325 is added to blk_1073741851_1036 (size=5056) 2024-11-14T14:55:36,788 INFO [M:0;da1c6afcd1f9:45313 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ec2f7adb7982446b91adeec27827ac3a 2024-11-14T14:55:36,794 DEBUG [M:0;da1c6afcd1f9:45313 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ddc70494fc5e4272bd980d468cbe4539 as hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ddc70494fc5e4272bd980d468cbe4539 2024-11-14T14:55:36,799 INFO [M:0;da1c6afcd1f9:45313 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ddc70494fc5e4272bd980d468cbe4539, entries=8, sequenceid=56, filesize=5.5 K 2024-11-14T14:55:36,800 DEBUG [M:0;da1c6afcd1f9:45313 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3a21f0758f544d0681b86f04ee18f45c as hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3a21f0758f544d0681b86f04ee18f45c 2024-11-14T14:55:36,805 INFO [M:0;da1c6afcd1f9:45313 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3a21f0758f544d0681b86f04ee18f45c, entries=6, sequenceid=56, filesize=6.0 K 2024-11-14T14:55:36,806 DEBUG [M:0;da1c6afcd1f9:45313 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2a7b367d2af5407e9473a3dc689bcb3b as hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2a7b367d2af5407e9473a3dc689bcb3b 2024-11-14T14:55:36,811 INFO [M:0;da1c6afcd1f9:45313 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2a7b367d2af5407e9473a3dc689bcb3b, entries=1, sequenceid=56, filesize=5.0 K 2024-11-14T14:55:36,812 DEBUG [M:0;da1c6afcd1f9:45313 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ec2f7adb7982446b91adeec27827ac3a as hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ec2f7adb7982446b91adeec27827ac3a 2024-11-14T14:55:36,817 INFO [M:0;da1c6afcd1f9:45313 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ec2f7adb7982446b91adeec27827ac3a, entries=1, sequenceid=56, filesize=4.9 K 2024-11-14T14:55:36,819 INFO [M:0;da1c6afcd1f9:45313 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 166ms, sequenceid=56, compaction requested=false 2024-11-14T14:55:36,824 INFO [M:0;da1c6afcd1f9:45313 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:55:36,824 DEBUG [M:0;da1c6afcd1f9:45313 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731596136652Disabling compacts and flushes for region at 1731596136652Disabling writes for close at 1731596136653 (+1 ms)Obtaining lock to block concurrent updates at 1731596136653Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731596136653Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1731596136653Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731596136665 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731596136665Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731596136683 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731596136683Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731596136710 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731596136727 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731596136727Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731596136740 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731596136755 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731596136755Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731596136766 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731596136781 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731596136782 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@383e706f: reopening flushed file at 1731596136793 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f8797e1: reopening flushed file at 1731596136799 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c90fae0: reopening flushed file at 1731596136805 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ba283e1: reopening flushed file at 1731596136811 (+6 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 166ms, sequenceid=56, compaction requested=false at 1731596136819 (+8 ms)Writing region close event to WAL at 1731596136824 (+5 ms)Closed at 1731596136824 2024-11-14T14:55:36,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:36,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:36,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:36,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:36,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:55:36,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44325 is added to blk_1073741847_1031 (size=757) 2024-11-14T14:55:36,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41267 is added to blk_1073741847_1031 (size=757) 2024-11-14T14:55:37,531 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:37,538 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:37,834 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:37,834 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:37,846 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:37,847 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:37,847 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:37,848 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:37,848 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:37,849 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:37,852 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:37,852 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:37,852 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:37,854 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:37,858 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:37,858 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:38,361 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-14T14:55:38,362 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:38,362 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:38,363 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:38,363 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:38,377 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:38,377 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:38,377 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:38,378 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:38,378 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:38,379 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:38,381 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:38,382 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:38,382 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:38,384 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:38,531 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:38,539 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:38,631 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741830_1015: GenerationStamp not matched, existing replica is blk_1073741830_1006 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-14T14:55:39,532 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:39,539 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:39,616 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:55:39,617 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T14:55:39,617 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T14:55:39,617 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-11-14T14:55:40,533 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:40,540 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:40,662 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/WALs/da1c6afcd1f9,45313,1731596105932/da1c6afcd1f9%2C45313%2C1731596105932.1731596106675 after 4001ms 2024-11-14T14:55:40,662 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/WALs/da1c6afcd1f9,45313,1731596105932/da1c6afcd1f9%2C45313%2C1731596105932.1731596106675 to hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/oldWALs/da1c6afcd1f9%2C45313%2C1731596105932.1731596106675 2024-11-14T14:55:40,665 INFO [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/MasterData/oldWALs/da1c6afcd1f9%2C45313%2C1731596105932.1731596106675 to hdfs://localhost:33315/user/jenkins/test-data/0c42ef9d-6815-21f0-bbc9-f605777778de/oldWALs/da1c6afcd1f9%2C45313%2C1731596105932.1731596106675$masterlocalwal$ 2024-11-14T14:55:40,665 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:55:40,665 INFO [M:0;da1c6afcd1f9:45313 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T14:55:40,665 INFO [M:0;da1c6afcd1f9:45313 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45313 2024-11-14T14:55:40,665 INFO [M:0;da1c6afcd1f9:45313 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:55:40,767 INFO [M:0;da1c6afcd1f9:45313 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:55:40,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:55:40,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45313-0x101a7472e0b0000, quorum=127.0.0.1:63452, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:55:40,770 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@22586a60{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:55:40,770 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5684bce2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:55:40,770 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:55:40,771 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4251f41a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:55:40,771 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1decdda3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/hadoop.log.dir/,STOPPED} 2024-11-14T14:55:40,772 WARN [BP-517382180-172.17.0.2-1731596104998 heartbeating to localhost/127.0.0.1:33315 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:55:40,772 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:55:40,772 WARN [BP-517382180-172.17.0.2-1731596104998 heartbeating to localhost/127.0.0.1:33315 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-517382180-172.17.0.2-1731596104998 (Datanode Uuid 1d51cfb9-dd20-4689-a9a3-b54aa021f596) service to localhost/127.0.0.1:33315 2024-11-14T14:55:40,772 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:55:40,773 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/data/data3/current/BP-517382180-172.17.0.2-1731596104998 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:55:40,773 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/data/data4/current/BP-517382180-172.17.0.2-1731596104998 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:55:40,773 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:55:40,775 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4eef3a93{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:55:40,775 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3d8da7be{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:55:40,776 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:55:40,776 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2378632b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:55:40,776 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20432799{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/hadoop.log.dir/,STOPPED} 2024-11-14T14:55:40,777 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:55:40,777 WARN [BP-517382180-172.17.0.2-1731596104998 heartbeating to localhost/127.0.0.1:33315 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:55:40,777 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:55:40,777 WARN [BP-517382180-172.17.0.2-1731596104998 heartbeating to localhost/127.0.0.1:33315 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-517382180-172.17.0.2-1731596104998 (Datanode Uuid 972bbbb2-749b-46dd-bc90-7ff3b222239e) service to localhost/127.0.0.1:33315 2024-11-14T14:55:40,778 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/data/data1/current/BP-517382180-172.17.0.2-1731596104998 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:55:40,778 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/cluster_23de681b-a65d-31da-17bf-4887e5d59a36/data/data2/current/BP-517382180-172.17.0.2-1731596104998 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:55:40,778 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:55:40,784 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@ba4770c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T14:55:40,784 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4702e786{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:55:40,785 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:55:40,785 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@10a92d53{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:55:40,785 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13fdd007{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/hadoop.log.dir/,STOPPED} 2024-11-14T14:55:40,791 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T14:55:40,810 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-14T14:55:40,819 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=182 (was 157) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33315 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-32-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33315 from jenkins.hfs.4 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33315 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33315 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33315 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:33315 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33315 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:33315 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 450) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=167 (was 115) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7448 (was 7727) 2024-11-14T14:55:40,827 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=182, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=167, ProcessCount=11, AvailableMemoryMB=7448 2024-11-14T14:55:40,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T14:55:40,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/hadoop.log.dir so I do NOT create it in target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754 2024-11-14T14:55:40,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0630bd08-9c00-efc8-b0fe-11c73517a27c/hadoop.tmp.dir so I do NOT create it in target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754 2024-11-14T14:55:40,828 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/cluster_18a57b98-5841-1b8f-fcd3-15b2160d9bd5, deleteOnExit=true 2024-11-14T14:55:40,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T14:55:40,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/test.cache.data in system properties and HBase conf 2024-11-14T14:55:40,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T14:55:40,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/hadoop.log.dir in system properties and HBase conf 2024-11-14T14:55:40,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T14:55:40,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T14:55:40,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T14:55:40,828 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T14:55:40,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T14:55:40,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T14:55:40,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T14:55:40,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T14:55:40,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T14:55:40,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T14:55:40,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T14:55:40,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T14:55:40,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T14:55:40,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/nfs.dump.dir in system properties and HBase conf 2024-11-14T14:55:40,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/java.io.tmpdir in system properties and HBase conf 2024-11-14T14:55:40,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T14:55:40,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T14:55:40,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T14:55:40,842 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-14T14:55:40,917 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:55:40,922 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:55:40,929 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:55:40,929 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:55:40,929 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:55:40,932 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:55:40,933 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@778c1ce5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:55:40,933 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ece2bc8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:55:41,067 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3ff21aec{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/java.io.tmpdir/jetty-localhost-36085-hadoop-hdfs-3_4_1-tests_jar-_-any-10614170905294636753/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T14:55:41,068 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@40e0483f{HTTP/1.1, (http/1.1)}{localhost:36085} 2024-11-14T14:55:41,068 INFO [Time-limited test {}] server.Server(415): Started @186690ms 2024-11-14T14:55:41,084 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-14T14:55:41,142 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:55:41,145 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:55:41,146 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:55:41,146 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:55:41,146 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T14:55:41,147 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a560185{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:55:41,147 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@168edd0a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:55:41,266 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@510b5e02{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/java.io.tmpdir/jetty-localhost-42673-hadoop-hdfs-3_4_1-tests_jar-_-any-1608527722291977414/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:55:41,266 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@59a5f2c5{HTTP/1.1, (http/1.1)}{localhost:42673} 2024-11-14T14:55:41,266 INFO [Time-limited test {}] server.Server(415): Started @186888ms 2024-11-14T14:55:41,268 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:55:41,302 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:55:41,305 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:55:41,306 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:55:41,306 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:55:41,306 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T14:55:41,307 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6c243e85{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:55:41,307 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4ed77c81{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:55:41,364 WARN [Thread-1648 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/cluster_18a57b98-5841-1b8f-fcd3-15b2160d9bd5/data/data1/current/BP-309921800-172.17.0.2-1731596140861/current, will proceed with Du for space computation calculation, 2024-11-14T14:55:41,364 WARN [Thread-1649 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/cluster_18a57b98-5841-1b8f-fcd3-15b2160d9bd5/data/data2/current/BP-309921800-172.17.0.2-1731596140861/current, will proceed with Du for space computation calculation, 2024-11-14T14:55:41,387 WARN [Thread-1627 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:55:41,391 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5d994bcf0827dbcd with lease ID 0x3a9e52d120ec67ac: Processing first storage report for DS-20a3344e-b52f-4610-9363-900326dffce6 from datanode DatanodeRegistration(127.0.0.1:46057, datanodeUuid=b10d73d4-de38-4304-9c3e-a50692aba180, infoPort=45483, infoSecurePort=0, ipcPort=35997, storageInfo=lv=-57;cid=testClusterID;nsid=238684674;c=1731596140861) 2024-11-14T14:55:41,391 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5d994bcf0827dbcd with lease ID 0x3a9e52d120ec67ac: from storage DS-20a3344e-b52f-4610-9363-900326dffce6 node DatanodeRegistration(127.0.0.1:46057, datanodeUuid=b10d73d4-de38-4304-9c3e-a50692aba180, infoPort=45483, infoSecurePort=0, ipcPort=35997, storageInfo=lv=-57;cid=testClusterID;nsid=238684674;c=1731596140861), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:55:41,391 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5d994bcf0827dbcd with lease ID 0x3a9e52d120ec67ac: Processing first storage report for DS-28d03546-d6fe-4549-9148-46b7ae70ce71 from datanode DatanodeRegistration(127.0.0.1:46057, datanodeUuid=b10d73d4-de38-4304-9c3e-a50692aba180, infoPort=45483, infoSecurePort=0, ipcPort=35997, storageInfo=lv=-57;cid=testClusterID;nsid=238684674;c=1731596140861) 2024-11-14T14:55:41,391 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5d994bcf0827dbcd with lease ID 0x3a9e52d120ec67ac: from storage DS-28d03546-d6fe-4549-9148-46b7ae70ce71 node DatanodeRegistration(127.0.0.1:46057, datanodeUuid=b10d73d4-de38-4304-9c3e-a50692aba180, infoPort=45483, infoSecurePort=0, ipcPort=35997, storageInfo=lv=-57;cid=testClusterID;nsid=238684674;c=1731596140861), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:55:41,431 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@16a85bb6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/java.io.tmpdir/jetty-localhost-37473-hadoop-hdfs-3_4_1-tests_jar-_-any-18051666195352855168/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:55:41,431 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5cac9c50{HTTP/1.1, (http/1.1)}{localhost:37473} 2024-11-14T14:55:41,431 INFO [Time-limited test {}] server.Server(415): Started @187053ms 2024-11-14T14:55:41,433 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:55:41,533 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:41,536 WARN [Thread-1674 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/cluster_18a57b98-5841-1b8f-fcd3-15b2160d9bd5/data/data3/current/BP-309921800-172.17.0.2-1731596140861/current, will proceed with Du for space computation calculation, 2024-11-14T14:55:41,536 WARN [Thread-1675 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/cluster_18a57b98-5841-1b8f-fcd3-15b2160d9bd5/data/data4/current/BP-309921800-172.17.0.2-1731596140861/current, will proceed with Du for space computation calculation, 2024-11-14T14:55:41,541 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:41,556 WARN [Thread-1663 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:55:41,558 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1effa912b397caa0 with lease ID 0x3a9e52d120ec67ad: Processing first storage report for DS-4bee740f-033f-45a7-a7ad-59f56833b551 from datanode DatanodeRegistration(127.0.0.1:39745, datanodeUuid=ba51a9d6-cf30-4d31-81ba-36a08a0dd761, infoPort=32811, infoSecurePort=0, ipcPort=33613, storageInfo=lv=-57;cid=testClusterID;nsid=238684674;c=1731596140861) 2024-11-14T14:55:41,558 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1effa912b397caa0 with lease ID 0x3a9e52d120ec67ad: from storage DS-4bee740f-033f-45a7-a7ad-59f56833b551 node DatanodeRegistration(127.0.0.1:39745, datanodeUuid=ba51a9d6-cf30-4d31-81ba-36a08a0dd761, infoPort=32811, infoSecurePort=0, ipcPort=33613, storageInfo=lv=-57;cid=testClusterID;nsid=238684674;c=1731596140861), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:55:41,558 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1effa912b397caa0 with lease ID 0x3a9e52d120ec67ad: Processing first storage report for DS-01e39d2a-bfc0-412d-bbce-5c35d7c2d571 from datanode DatanodeRegistration(127.0.0.1:39745, datanodeUuid=ba51a9d6-cf30-4d31-81ba-36a08a0dd761, infoPort=32811, infoSecurePort=0, ipcPort=33613, storageInfo=lv=-57;cid=testClusterID;nsid=238684674;c=1731596140861) 2024-11-14T14:55:41,558 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1effa912b397caa0 with lease ID 0x3a9e52d120ec67ad: from storage DS-01e39d2a-bfc0-412d-bbce-5c35d7c2d571 node DatanodeRegistration(127.0.0.1:39745, datanodeUuid=ba51a9d6-cf30-4d31-81ba-36a08a0dd761, infoPort=32811, infoSecurePort=0, ipcPort=33613, storageInfo=lv=-57;cid=testClusterID;nsid=238684674;c=1731596140861), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:55:41,658 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754 2024-11-14T14:55:41,661 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/cluster_18a57b98-5841-1b8f-fcd3-15b2160d9bd5/zookeeper_0, clientPort=57943, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/cluster_18a57b98-5841-1b8f-fcd3-15b2160d9bd5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/cluster_18a57b98-5841-1b8f-fcd3-15b2160d9bd5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T14:55:41,662 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57943 2024-11-14T14:55:41,663 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:55:41,664 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:55:41,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:55:41,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:55:41,679 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8 with version=8 2024-11-14T14:55:41,679 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/hbase-staging 2024-11-14T14:55:41,681 INFO [Time-limited test {}] client.ConnectionUtils(128): master/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:55:41,681 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:55:41,681 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:55:41,681 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:55:41,681 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:55:41,681 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:55:41,681 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T14:55:41,682 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:55:41,682 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40583 2024-11-14T14:55:41,684 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40583 connecting to ZooKeeper ensemble=127.0.0.1:57943 2024-11-14T14:55:41,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:405830x0, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:55:41,690 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40583-0x101a747b9c50000 connected 2024-11-14T14:55:41,707 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:55:41,709 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:55:41,711 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:55:41,712 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8, hbase.cluster.distributed=false 2024-11-14T14:55:41,713 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:55:41,714 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40583 2024-11-14T14:55:41,714 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40583 2024-11-14T14:55:41,714 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40583 2024-11-14T14:55:41,715 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40583 2024-11-14T14:55:41,715 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40583 2024-11-14T14:55:41,730 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:55:41,731 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:55:41,731 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:55:41,731 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:55:41,731 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:55:41,731 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:55:41,731 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T14:55:41,731 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:55:41,732 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34489 2024-11-14T14:55:41,733 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34489 connecting to ZooKeeper ensemble=127.0.0.1:57943 2024-11-14T14:55:41,733 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:55:41,735 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:55:41,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:344890x0, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:55:41,740 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:344890x0, quorum=127.0.0.1:57943, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:55:41,740 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34489-0x101a747b9c50001 connected 2024-11-14T14:55:41,740 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T14:55:41,741 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T14:55:41,741 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T14:55:41,742 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:55:41,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34489 2024-11-14T14:55:41,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34489 2024-11-14T14:55:41,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34489 2024-11-14T14:55:41,744 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34489 2024-11-14T14:55:41,744 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34489 2024-11-14T14:55:41,756 DEBUG [M:0;da1c6afcd1f9:40583 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;da1c6afcd1f9:40583 2024-11-14T14:55:41,756 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/da1c6afcd1f9,40583,1731596141681 2024-11-14T14:55:41,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:55:41,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:55:41,759 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/da1c6afcd1f9,40583,1731596141681 2024-11-14T14:55:41,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T14:55:41,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:41,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:41,762 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T14:55:41,762 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/da1c6afcd1f9,40583,1731596141681 from backup master directory 2024-11-14T14:55:41,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/da1c6afcd1f9,40583,1731596141681 2024-11-14T14:55:41,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:55:41,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:55:41,764 WARN [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:55:41,764 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=da1c6afcd1f9,40583,1731596141681 2024-11-14T14:55:41,768 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/hbase.id] with ID: ead2bf4c-88dc-49e5-8e01-e80015964285 2024-11-14T14:55:41,768 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/.tmp/hbase.id 2024-11-14T14:55:41,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:55:41,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:55:41,776 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/.tmp/hbase.id]:[hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/hbase.id] 2024-11-14T14:55:41,789 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:55:41,789 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T14:55:41,791 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-14T14:55:41,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:41,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:41,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:55:41,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:55:41,806 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T14:55:41,806 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T14:55:41,807 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:55:41,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:55:41,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:55:41,822 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store 2024-11-14T14:55:41,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:55:41,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:55:41,830 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:55:41,830 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:55:41,830 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:55:41,830 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:55:41,830 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:55:41,830 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:55:41,830 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:55:41,830 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731596141830Disabling compacts and flushes for region at 1731596141830Disabling writes for close at 1731596141830Writing region close event to WAL at 1731596141830Closed at 1731596141830 2024-11-14T14:55:41,831 WARN [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/.initializing 2024-11-14T14:55:41,831 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/WALs/da1c6afcd1f9,40583,1731596141681 2024-11-14T14:55:41,834 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C40583%2C1731596141681, suffix=, logDir=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/WALs/da1c6afcd1f9,40583,1731596141681, archiveDir=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/oldWALs, maxLogs=10 2024-11-14T14:55:41,834 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C40583%2C1731596141681.1731596141834 2024-11-14T14:55:41,840 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/WALs/da1c6afcd1f9,40583,1731596141681/da1c6afcd1f9%2C40583%2C1731596141681.1731596141834 2024-11-14T14:55:41,841 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:32811:32811),(127.0.0.1/127.0.0.1:45483:45483)] 2024-11-14T14:55:41,842 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:55:41,842 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:55:41,842 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:41,842 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:41,844 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:41,845 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T14:55:41,845 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:41,845 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:55:41,846 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:41,847 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T14:55:41,847 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:41,847 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:55:41,847 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:41,848 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T14:55:41,848 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:41,849 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:55:41,849 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:41,850 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T14:55:41,850 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:41,851 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:55:41,851 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:41,851 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:41,852 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:41,853 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:41,853 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:41,854 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T14:55:41,855 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:55:41,857 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:55:41,858 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=869878, jitterRate=0.106108158826828}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T14:55:41,858 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731596141842Initializing all the Stores at 1731596141843 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596141843Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596141843Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596141843Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596141843Cleaning up temporary data from old regions at 1731596141853 (+10 ms)Region opened successfully at 1731596141858 (+5 ms) 2024-11-14T14:55:41,858 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T14:55:41,865 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5beb5482, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:55:41,866 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T14:55:41,866 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T14:55:41,866 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T14:55:41,867 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:55:41,867 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:55:41,868 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-14T14:55:41,868 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T14:55:41,870 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T14:55:41,871 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T14:55:41,873 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T14:55:41,873 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T14:55:41,874 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T14:55:41,876 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T14:55:41,876 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T14:55:41,877 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T14:55:41,879 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T14:55:41,879 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T14:55:41,881 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T14:55:41,883 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T14:55:41,884 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T14:55:41,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:55:41,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:55:41,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:41,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:41,887 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=da1c6afcd1f9,40583,1731596141681, sessionid=0x101a747b9c50000, setting cluster-up flag (Was=false) 2024-11-14T14:55:41,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:41,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:41,895 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T14:55:41,896 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da1c6afcd1f9,40583,1731596141681 2024-11-14T14:55:41,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:41,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:41,905 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T14:55:41,906 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da1c6afcd1f9,40583,1731596141681 2024-11-14T14:55:41,908 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T14:55:41,909 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:55:41,910 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T14:55:41,910 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T14:55:41,910 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: da1c6afcd1f9,40583,1731596141681 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T14:55:41,912 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:55:41,912 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:55:41,912 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:55:41,912 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:55:41,912 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/da1c6afcd1f9:0, corePoolSize=10, maxPoolSize=10 2024-11-14T14:55:41,912 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:41,912 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:55:41,912 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:41,913 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731596171913 2024-11-14T14:55:41,913 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T14:55:41,913 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T14:55:41,913 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T14:55:41,914 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T14:55:41,914 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T14:55:41,914 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T14:55:41,914 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:41,914 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T14:55:41,914 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T14:55:41,914 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:55:41,914 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T14:55:41,914 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T14:55:41,915 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T14:55:41,915 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T14:55:41,915 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731596141915,5,FailOnTimeoutGroup] 2024-11-14T14:55:41,915 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731596141915,5,FailOnTimeoutGroup] 2024-11-14T14:55:41,915 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:41,915 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T14:55:41,915 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:41,915 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:41,915 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:41,916 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T14:55:41,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:55:41,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:55:41,927 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T14:55:41,927 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8 2024-11-14T14:55:41,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:55:41,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:55:41,946 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.HRegionServer(746): ClusterId : ead2bf4c-88dc-49e5-8e01-e80015964285 2024-11-14T14:55:41,946 DEBUG [RS:0;da1c6afcd1f9:34489 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T14:55:41,948 DEBUG [RS:0;da1c6afcd1f9:34489 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T14:55:41,948 DEBUG [RS:0;da1c6afcd1f9:34489 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T14:55:41,950 DEBUG [RS:0;da1c6afcd1f9:34489 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T14:55:41,950 DEBUG [RS:0;da1c6afcd1f9:34489 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c61452, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:55:41,963 DEBUG [RS:0;da1c6afcd1f9:34489 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;da1c6afcd1f9:34489 2024-11-14T14:55:41,963 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T14:55:41,963 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T14:55:41,963 DEBUG [RS:0;da1c6afcd1f9:34489 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T14:55:41,964 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.HRegionServer(2659): reportForDuty to master=da1c6afcd1f9,40583,1731596141681 with port=34489, startcode=1731596141730 2024-11-14T14:55:41,964 DEBUG [RS:0;da1c6afcd1f9:34489 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T14:55:41,966 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50181, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T14:55:41,967 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40583 {}] master.ServerManager(363): Checking decommissioned status of RegionServer da1c6afcd1f9,34489,1731596141730 2024-11-14T14:55:41,967 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40583 {}] master.ServerManager(517): Registering regionserver=da1c6afcd1f9,34489,1731596141730 2024-11-14T14:55:41,969 DEBUG [RS:0;da1c6afcd1f9:34489 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8 2024-11-14T14:55:41,969 DEBUG [RS:0;da1c6afcd1f9:34489 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37087 2024-11-14T14:55:41,969 DEBUG [RS:0;da1c6afcd1f9:34489 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T14:55:41,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:55:41,972 DEBUG [RS:0;da1c6afcd1f9:34489 {}] zookeeper.ZKUtil(111): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/da1c6afcd1f9,34489,1731596141730 2024-11-14T14:55:41,972 WARN [RS:0;da1c6afcd1f9:34489 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:55:41,972 INFO [RS:0;da1c6afcd1f9:34489 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:55:41,972 DEBUG [RS:0;da1c6afcd1f9:34489 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/WALs/da1c6afcd1f9,34489,1731596141730 2024-11-14T14:55:41,972 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [da1c6afcd1f9,34489,1731596141730] 2024-11-14T14:55:41,976 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T14:55:41,978 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T14:55:41,978 INFO [RS:0;da1c6afcd1f9:34489 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T14:55:41,978 INFO [RS:0;da1c6afcd1f9:34489 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:41,978 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T14:55:41,979 INFO [RS:0;da1c6afcd1f9:34489 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T14:55:41,979 INFO [RS:0;da1c6afcd1f9:34489 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:41,979 DEBUG [RS:0;da1c6afcd1f9:34489 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:41,979 DEBUG [RS:0;da1c6afcd1f9:34489 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:41,979 DEBUG [RS:0;da1c6afcd1f9:34489 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:41,979 DEBUG [RS:0;da1c6afcd1f9:34489 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:41,979 DEBUG [RS:0;da1c6afcd1f9:34489 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:41,980 DEBUG [RS:0;da1c6afcd1f9:34489 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:55:41,980 DEBUG [RS:0;da1c6afcd1f9:34489 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:41,980 DEBUG [RS:0;da1c6afcd1f9:34489 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:41,980 DEBUG [RS:0;da1c6afcd1f9:34489 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:41,980 DEBUG [RS:0;da1c6afcd1f9:34489 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:41,980 DEBUG [RS:0;da1c6afcd1f9:34489 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:41,980 DEBUG [RS:0;da1c6afcd1f9:34489 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:55:41,980 DEBUG [RS:0;da1c6afcd1f9:34489 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:55:41,980 DEBUG [RS:0;da1c6afcd1f9:34489 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:55:41,980 INFO [RS:0;da1c6afcd1f9:34489 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:41,980 INFO [RS:0;da1c6afcd1f9:34489 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:41,980 INFO [RS:0;da1c6afcd1f9:34489 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:41,981 INFO [RS:0;da1c6afcd1f9:34489 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:41,981 INFO [RS:0;da1c6afcd1f9:34489 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:41,981 INFO [RS:0;da1c6afcd1f9:34489 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,34489,1731596141730-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:55:41,996 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T14:55:41,996 INFO [RS:0;da1c6afcd1f9:34489 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,34489,1731596141730-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:41,996 INFO [RS:0;da1c6afcd1f9:34489 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:41,996 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.Replication(171): da1c6afcd1f9,34489,1731596141730 started 2024-11-14T14:55:42,011 INFO [RS:0;da1c6afcd1f9:34489 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:42,011 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.HRegionServer(1482): Serving as da1c6afcd1f9,34489,1731596141730, RpcServer on da1c6afcd1f9/172.17.0.2:34489, sessionid=0x101a747b9c50001 2024-11-14T14:55:42,011 DEBUG [RS:0;da1c6afcd1f9:34489 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T14:55:42,011 DEBUG [RS:0;da1c6afcd1f9:34489 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager da1c6afcd1f9,34489,1731596141730 2024-11-14T14:55:42,011 DEBUG [RS:0;da1c6afcd1f9:34489 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,34489,1731596141730' 2024-11-14T14:55:42,011 DEBUG [RS:0;da1c6afcd1f9:34489 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T14:55:42,012 DEBUG [RS:0;da1c6afcd1f9:34489 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T14:55:42,013 DEBUG [RS:0;da1c6afcd1f9:34489 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T14:55:42,013 DEBUG [RS:0;da1c6afcd1f9:34489 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T14:55:42,013 DEBUG [RS:0;da1c6afcd1f9:34489 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager da1c6afcd1f9,34489,1731596141730 2024-11-14T14:55:42,013 DEBUG [RS:0;da1c6afcd1f9:34489 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,34489,1731596141730' 2024-11-14T14:55:42,013 DEBUG [RS:0;da1c6afcd1f9:34489 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T14:55:42,013 DEBUG [RS:0;da1c6afcd1f9:34489 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T14:55:42,014 DEBUG [RS:0;da1c6afcd1f9:34489 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T14:55:42,014 INFO [RS:0;da1c6afcd1f9:34489 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T14:55:42,014 INFO [RS:0;da1c6afcd1f9:34489 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T14:55:42,116 INFO [RS:0;da1c6afcd1f9:34489 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C34489%2C1731596141730, suffix=, logDir=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/WALs/da1c6afcd1f9,34489,1731596141730, archiveDir=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/oldWALs, maxLogs=32 2024-11-14T14:55:42,117 INFO [RS:0;da1c6afcd1f9:34489 {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C34489%2C1731596141730.1731596142117 2024-11-14T14:55:42,123 INFO [RS:0;da1c6afcd1f9:34489 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/WALs/da1c6afcd1f9,34489,1731596141730/da1c6afcd1f9%2C34489%2C1731596141730.1731596142117 2024-11-14T14:55:42,124 DEBUG [RS:0;da1c6afcd1f9:34489 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45483:45483),(127.0.0.1/127.0.0.1:32811:32811)] 2024-11-14T14:55:42,336 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:55:42,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:55:42,339 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:55:42,339 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:42,340 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:55:42,340 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:55:42,341 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:55:42,341 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:42,342 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:55:42,342 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:55:42,343 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:55:42,343 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:42,344 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:55:42,344 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:55:42,345 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:55:42,345 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:42,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:55:42,346 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:55:42,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740 2024-11-14T14:55:42,347 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740 2024-11-14T14:55:42,349 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:55:42,349 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:55:42,349 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T14:55:42,351 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:55:42,353 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:55:42,354 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=878978, jitterRate=0.11767925322055817}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T14:55:42,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731596142336Initializing all the Stores at 1731596142337 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596142337Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596142337Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596142337Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596142337Cleaning up temporary data from old regions at 1731596142349 (+12 ms)Region opened successfully at 1731596142354 (+5 ms) 2024-11-14T14:55:42,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:55:42,355 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:55:42,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:55:42,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:55:42,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:55:42,355 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:55:42,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731596142355Disabling compacts and flushes for region at 1731596142355Disabling writes for close at 1731596142355Writing region close event to WAL at 1731596142355Closed at 1731596142355 2024-11-14T14:55:42,357 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:55:42,357 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T14:55:42,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T14:55:42,359 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:55:42,360 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T14:55:42,511 DEBUG [da1c6afcd1f9:40583 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T14:55:42,511 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=da1c6afcd1f9,34489,1731596141730 2024-11-14T14:55:42,513 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,34489,1731596141730, state=OPENING 2024-11-14T14:55:42,515 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T14:55:42,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:42,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:55:42,518 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:55:42,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:55:42,518 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:55:42,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,34489,1731596141730}] 2024-11-14T14:55:42,534 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:42,541 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:42,672 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T14:55:42,674 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51059, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T14:55:42,677 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T14:55:42,678 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:55:42,680 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C34489%2C1731596141730.meta, suffix=.meta, logDir=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/WALs/da1c6afcd1f9,34489,1731596141730, archiveDir=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/oldWALs, maxLogs=32 2024-11-14T14:55:42,680 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C34489%2C1731596141730.meta.1731596142680.meta 2024-11-14T14:55:42,685 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/WALs/da1c6afcd1f9,34489,1731596141730/da1c6afcd1f9%2C34489%2C1731596141730.meta.1731596142680.meta 2024-11-14T14:55:42,686 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45483:45483),(127.0.0.1/127.0.0.1:32811:32811)] 2024-11-14T14:55:42,687 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:55:42,687 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T14:55:42,687 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T14:55:42,688 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T14:55:42,688 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T14:55:42,688 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:55:42,688 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T14:55:42,688 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T14:55:42,689 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:55:42,690 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:55:42,690 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:42,691 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:55:42,691 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:55:42,691 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:55:42,691 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:42,692 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:55:42,692 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:55:42,693 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:55:42,693 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:42,693 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:55:42,693 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:55:42,694 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:55:42,694 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:42,694 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:55:42,694 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:55:42,695 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740 2024-11-14T14:55:42,696 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740 2024-11-14T14:55:42,698 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:55:42,698 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:55:42,698 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T14:55:42,699 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:55:42,700 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=845845, jitterRate=0.07554768025875092}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T14:55:42,700 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T14:55:42,701 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731596142688Writing region info on filesystem at 1731596142688Initializing all the Stores at 1731596142689 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596142689Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596142689Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596142689Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596142689Cleaning up temporary data from old regions at 1731596142698 (+9 ms)Running coprocessor post-open hooks at 1731596142700 (+2 ms)Region opened successfully at 1731596142701 (+1 ms) 2024-11-14T14:55:42,702 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731596142671 2024-11-14T14:55:42,705 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T14:55:42,705 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T14:55:42,706 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,34489,1731596141730 2024-11-14T14:55:42,707 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,34489,1731596141730, state=OPEN 2024-11-14T14:55:42,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:55:42,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:55:42,712 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,34489,1731596141730 2024-11-14T14:55:42,712 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:55:42,712 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:55:42,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T14:55:42,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,34489,1731596141730 in 194 msec 2024-11-14T14:55:42,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T14:55:42,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 358 msec 2024-11-14T14:55:42,719 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:55:42,719 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T14:55:42,720 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:55:42,721 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,34489,1731596141730, seqNum=-1] 2024-11-14T14:55:42,721 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:55:42,722 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34247, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:55:42,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 818 msec 2024-11-14T14:55:42,728 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731596142728, completionTime=-1 2024-11-14T14:55:42,728 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T14:55:42,728 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:55:42,730 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-14T14:55:42,730 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731596202730 2024-11-14T14:55:42,730 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731596262730 2024-11-14T14:55:42,730 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-11-14T14:55:42,730 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40583,1731596141681-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:42,730 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40583,1731596141681-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:42,730 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40583,1731596141681-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:42,731 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-da1c6afcd1f9:40583, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:42,731 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:42,731 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:42,732 DEBUG [master/da1c6afcd1f9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T14:55:42,735 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.971sec 2024-11-14T14:55:42,735 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T14:55:42,735 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T14:55:42,735 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T14:55:42,735 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T14:55:42,735 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T14:55:42,735 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40583,1731596141681-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:55:42,735 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40583,1731596141681-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T14:55:42,738 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T14:55:42,738 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T14:55:42,738 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,40583,1731596141681-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:55:42,747 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a3c6b7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:55:42,747 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request da1c6afcd1f9,40583,-1 for getting cluster id 2024-11-14T14:55:42,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T14:55:42,749 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ead2bf4c-88dc-49e5-8e01-e80015964285' 2024-11-14T14:55:42,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T14:55:42,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ead2bf4c-88dc-49e5-8e01-e80015964285" 2024-11-14T14:55:42,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67d338fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:55:42,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [da1c6afcd1f9,40583,-1] 2024-11-14T14:55:42,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T14:55:42,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:55:42,751 INFO [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T14:55:42,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38b8be0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:55:42,753 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:55:42,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,34489,1731596141730, seqNum=-1] 2024-11-14T14:55:42,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:55:42,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:55:42,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=da1c6afcd1f9,40583,1731596141681 2024-11-14T14:55:42,757 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:55:42,759 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-14T14:55:42,760 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T14:55:42,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is da1c6afcd1f9,40583,1731596141681 2024-11-14T14:55:42,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4ca2c804 2024-11-14T14:55:42,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T14:55:42,762 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T14:55:42,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-14T14:55:42,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-14T14:55:42,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T14:55:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-14T14:55:42,766 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T14:55:42,766 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:42,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling" procId is: 4 2024-11-14T14:55:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:55:42,767 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T14:55:42,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741835_1011 (size=405) 2024-11-14T14:55:42,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741835_1011 (size=405) 2024-11-14T14:55:42,777 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 25108625f673cf8c0c90412d406225a3, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8 2024-11-14T14:55:42,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741836_1012 (size=88) 2024-11-14T14:55:42,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741836_1012 (size=88) 2024-11-14T14:55:42,787 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:55:42,787 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 25108625f673cf8c0c90412d406225a3, disabling compactions & flushes 2024-11-14T14:55:42,787 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:55:42,787 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:55:42,787 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. after waiting 0 ms 2024-11-14T14:55:42,787 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:55:42,787 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:55:42,787 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 25108625f673cf8c0c90412d406225a3: Waiting for close lock at 1731596142787Disabling compacts and flushes for region at 1731596142787Disabling writes for close at 1731596142787Writing region close event to WAL at 1731596142787Closed at 1731596142787 2024-11-14T14:55:42,789 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T14:55:42,789 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731596142789"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731596142789"}]},"ts":"1731596142789"} 2024-11-14T14:55:42,792 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T14:55:42,793 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T14:55:42,793 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731596142793"}]},"ts":"1731596142793"} 2024-11-14T14:55:42,795 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta 2024-11-14T14:55:42,796 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=25108625f673cf8c0c90412d406225a3, ASSIGN}] 2024-11-14T14:55:42,797 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=25108625f673cf8c0c90412d406225a3, ASSIGN 2024-11-14T14:55:42,798 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=25108625f673cf8c0c90412d406225a3, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,34489,1731596141730; forceNewPlan=false, retain=false 2024-11-14T14:55:42,949 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=25108625f673cf8c0c90412d406225a3, regionState=OPENING, regionLocation=da1c6afcd1f9,34489,1731596141730 2024-11-14T14:55:42,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=25108625f673cf8c0c90412d406225a3, ASSIGN because future has completed 2024-11-14T14:55:42,952 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25108625f673cf8c0c90412d406225a3, server=da1c6afcd1f9,34489,1731596141730}] 2024-11-14T14:55:43,109 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:55:43,110 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 25108625f673cf8c0c90412d406225a3, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:55:43,110 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 25108625f673cf8c0c90412d406225a3 2024-11-14T14:55:43,110 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:55:43,110 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 25108625f673cf8c0c90412d406225a3 2024-11-14T14:55:43,110 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 25108625f673cf8c0c90412d406225a3 2024-11-14T14:55:43,112 INFO [StoreOpener-25108625f673cf8c0c90412d406225a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 25108625f673cf8c0c90412d406225a3 2024-11-14T14:55:43,113 INFO [StoreOpener-25108625f673cf8c0c90412d406225a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 25108625f673cf8c0c90412d406225a3 columnFamilyName info 2024-11-14T14:55:43,113 DEBUG [StoreOpener-25108625f673cf8c0c90412d406225a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:55:43,114 INFO [StoreOpener-25108625f673cf8c0c90412d406225a3-1 {}] regionserver.HStore(327): Store=25108625f673cf8c0c90412d406225a3/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:55:43,114 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 25108625f673cf8c0c90412d406225a3 2024-11-14T14:55:43,115 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3 2024-11-14T14:55:43,115 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3 2024-11-14T14:55:43,116 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 25108625f673cf8c0c90412d406225a3 2024-11-14T14:55:43,116 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 25108625f673cf8c0c90412d406225a3 2024-11-14T14:55:43,118 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 25108625f673cf8c0c90412d406225a3 2024-11-14T14:55:43,120 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:55:43,120 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 25108625f673cf8c0c90412d406225a3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=743544, jitterRate=-0.05453580617904663}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:55:43,120 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 25108625f673cf8c0c90412d406225a3 2024-11-14T14:55:43,121 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 25108625f673cf8c0c90412d406225a3: Running coprocessor pre-open hook at 1731596143110Writing region info on filesystem at 1731596143110Initializing all the Stores at 1731596143111 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596143111Cleaning up temporary data from old regions at 1731596143116 (+5 ms)Running coprocessor post-open hooks at 1731596143120 (+4 ms)Region opened successfully at 1731596143121 (+1 ms) 2024-11-14T14:55:43,122 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3., pid=6, masterSystemTime=1731596143105 2024-11-14T14:55:43,125 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:55:43,125 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:55:43,126 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=25108625f673cf8c0c90412d406225a3, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,34489,1731596141730 2024-11-14T14:55:43,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 25108625f673cf8c0c90412d406225a3, server=da1c6afcd1f9,34489,1731596141730 because future has completed 2024-11-14T14:55:43,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T14:55:43,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 25108625f673cf8c0c90412d406225a3, server=da1c6afcd1f9,34489,1731596141730 in 178 msec 2024-11-14T14:55:43,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T14:55:43,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=25108625f673cf8c0c90412d406225a3, ASSIGN in 337 msec 2024-11-14T14:55:43,137 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T14:55:43,137 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731596143137"}]},"ts":"1731596143137"} 2024-11-14T14:55:43,139 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta 2024-11-14T14:55:43,140 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T14:55:43,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 377 msec 2024-11-14T14:55:43,535 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:43,542 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:44,535 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:44,542 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:45,119 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-14T14:55:45,120 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:45,120 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:45,120 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:45,121 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:45,121 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:45,121 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:45,137 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:45,138 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:45,138 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:45,139 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:45,139 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:45,139 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:45,143 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:45,143 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:45,143 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:45,146 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:55:45,536 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:45,543 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:46,537 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:46,543 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:47,538 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:47,544 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:47,976 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T14:55:47,976 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling' 2024-11-14T14:55:48,538 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:48,545 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:49,539 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:49,545 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:49,616 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T14:55:49,616 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T14:55:49,617 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:55:49,617 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T14:55:49,617 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T14:55:49,617 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T14:55:49,618 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-14T14:55:49,618 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer 2024-11-14T14:55:50,539 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:50,546 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:51,540 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:51,546 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:52,541 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:52,547 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:55:52,857 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-14T14:55:52,857 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100 2024-11-14T14:55:52,861 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-14T14:55:52,861 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:55:52,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3., hostname=da1c6afcd1f9,34489,1731596141730, seqNum=2] 2024-11-14T14:55:52,871 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-14T14:55:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-14T14:55:52,878 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-14T14:55:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T14:55:52,880 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T14:55:52,881 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T14:55:53,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34489 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-14T14:55:53,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:55:53,043 INFO [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 25108625f673cf8c0c90412d406225a3 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-14T14:55:53,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/.tmp/info/1a7182e7f832450686d08a824e8297ae is 1080, key is row0001/info:/1731596152865/Put/seqid=0 2024-11-14T14:55:53,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741837_1013 (size=6033) 2024-11-14T14:55:53,066 INFO [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/.tmp/info/1a7182e7f832450686d08a824e8297ae 2024-11-14T14:55:53,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741837_1013 (size=6033) 2024-11-14T14:55:53,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/.tmp/info/1a7182e7f832450686d08a824e8297ae as hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/1a7182e7f832450686d08a824e8297ae 2024-11-14T14:55:53,082 INFO [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/1a7182e7f832450686d08a824e8297ae, entries=1, sequenceid=5, filesize=5.9 K 2024-11-14T14:55:53,083 INFO [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 25108625f673cf8c0c90412d406225a3 in 41ms, sequenceid=5, compaction requested=false 2024-11-14T14:55:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 25108625f673cf8c0c90412d406225a3: 2024-11-14T14:55:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:55:53,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-14T14:55:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-14T14:55:53,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T14:55:53,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 208 msec 2024-11-14T14:55:53,095 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 220 msec 2024-11-14T14:55:53,541 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:53,547 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:54,542 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:54,548 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:55,543 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:55,548 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:56,543 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:56,549 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:57,544 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:57,550 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:58,544 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:58,550 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:59,545 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:55:59,551 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:00,546 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:00,551 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:01,546 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:01,552 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:02,547 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:02,552 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-14T14:56:02,987 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-14T14:56:02,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-14T14:56:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-14T14:56:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T14:56:02,993 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-14T14:56:02,994 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T14:56:02,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T14:56:03,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34489 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-14T14:56:03,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:56:03,147 INFO [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 25108625f673cf8c0c90412d406225a3 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-14T14:56:03,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/.tmp/info/ef8e0612a642481abb2d479a284f04b3 is 1080, key is row0002/info:/1731596162988/Put/seqid=0 2024-11-14T14:56:03,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741838_1014 (size=6033) 2024-11-14T14:56:03,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741838_1014 (size=6033) 2024-11-14T14:56:03,169 INFO [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/.tmp/info/ef8e0612a642481abb2d479a284f04b3 2024-11-14T14:56:03,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/.tmp/info/ef8e0612a642481abb2d479a284f04b3 as hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/ef8e0612a642481abb2d479a284f04b3 2024-11-14T14:56:03,183 INFO [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/ef8e0612a642481abb2d479a284f04b3, entries=1, sequenceid=9, filesize=5.9 K 2024-11-14T14:56:03,184 INFO [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 25108625f673cf8c0c90412d406225a3 in 37ms, sequenceid=9, compaction requested=false 2024-11-14T14:56:03,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 25108625f673cf8c0c90412d406225a3: 2024-11-14T14:56:03,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:56:03,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-14T14:56:03,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-14T14:56:03,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-14T14:56:03,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 192 msec 2024-11-14T14:56:03,192 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 200 msec 2024-11-14T14:56:03,547 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:03,553 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:04,548 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:04,553 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:05,549 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:05,549 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 after 68050ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor191.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:56:05,554 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:05,554 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta after 68039ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor191.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T14:56:06,550 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:06,554 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:07,551 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:07,555 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:08,551 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:08,555 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:09,552 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:09,556 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:10,553 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:10,556 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:11,553 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:11,557 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:11,658 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T14:56:12,554 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:12,557 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:13,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-14T14:56:13,086 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-14T14:56:13,090 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C34489%2C1731596141730.1731596173089 2024-11-14T14:56:13,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:13,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:13,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:13,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:13,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:13,096 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/WALs/da1c6afcd1f9,34489,1731596141730/da1c6afcd1f9%2C34489%2C1731596141730.1731596142117 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/WALs/da1c6afcd1f9,34489,1731596141730/da1c6afcd1f9%2C34489%2C1731596141730.1731596173089 2024-11-14T14:56:13,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741833_1009 (size=5546) 2024-11-14T14:56:13,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741833_1009 (size=5546) 2024-11-14T14:56:13,100 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45483:45483),(127.0.0.1/127.0.0.1:32811:32811)] 2024-11-14T14:56:13,101 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-14T14:56:13,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-14T14:56:13,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T14:56:13,104 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-14T14:56:13,105 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T14:56:13,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T14:56:13,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34489 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-14T14:56:13,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:56:13,259 INFO [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 25108625f673cf8c0c90412d406225a3 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-14T14:56:13,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/.tmp/info/a3b99ec727774299b09c69fb66fa0803 is 1080, key is row0003/info:/1731596173088/Put/seqid=0 2024-11-14T14:56:13,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741840_1016 (size=6033) 2024-11-14T14:56:13,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741840_1016 (size=6033) 2024-11-14T14:56:13,270 INFO [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/.tmp/info/a3b99ec727774299b09c69fb66fa0803 2024-11-14T14:56:13,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/.tmp/info/a3b99ec727774299b09c69fb66fa0803 as hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/a3b99ec727774299b09c69fb66fa0803 2024-11-14T14:56:13,284 INFO [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/a3b99ec727774299b09c69fb66fa0803, entries=1, sequenceid=13, filesize=5.9 K 2024-11-14T14:56:13,285 INFO [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 25108625f673cf8c0c90412d406225a3 in 26ms, sequenceid=13, compaction requested=true 2024-11-14T14:56:13,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 25108625f673cf8c0c90412d406225a3: 2024-11-14T14:56:13,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:56:13,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-14T14:56:13,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-14T14:56:13,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-14T14:56:13,290 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 182 msec 2024-11-14T14:56:13,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 190 msec 2024-11-14T14:56:13,554 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:13,558 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:14,555 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:14,558 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:15,556 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:15,559 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:16,556 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:16,560 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:17,557 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:17,560 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:18,558 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:18,560 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:19,558 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:19,561 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:20,559 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:20,561 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:21,559 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:21,562 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:22,560 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:22,563 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-14T14:56:23,137 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-14T14:56:23,137 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T14:56:23,139 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T14:56:23,139 DEBUG [Time-limited test {}] regionserver.HStore(1541): 25108625f673cf8c0c90412d406225a3/info is initiating minor compaction (all files) 2024-11-14T14:56:23,139 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T14:56:23,139 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:23,139 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 25108625f673cf8c0c90412d406225a3/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:56:23,139 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/1a7182e7f832450686d08a824e8297ae, hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/ef8e0612a642481abb2d479a284f04b3, hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/a3b99ec727774299b09c69fb66fa0803] into tmpdir=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/.tmp, totalSize=17.7 K 2024-11-14T14:56:23,140 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1a7182e7f832450686d08a824e8297ae, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731596152865 2024-11-14T14:56:23,140 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ef8e0612a642481abb2d479a284f04b3, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1731596162988 2024-11-14T14:56:23,141 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a3b99ec727774299b09c69fb66fa0803, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731596173088 2024-11-14T14:56:23,161 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 25108625f673cf8c0c90412d406225a3#info#compaction#46 average throughput is 3.08 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T14:56:23,161 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/.tmp/info/f082587d97b64b5b8d0b27052604032a is 1080, key is row0001/info:/1731596152865/Put/seqid=0 2024-11-14T14:56:23,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741841_1017 (size=8296) 2024-11-14T14:56:23,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741841_1017 (size=8296) 2024-11-14T14:56:23,173 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/.tmp/info/f082587d97b64b5b8d0b27052604032a as hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/f082587d97b64b5b8d0b27052604032a 2024-11-14T14:56:23,180 INFO [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 25108625f673cf8c0c90412d406225a3/info of 25108625f673cf8c0c90412d406225a3 into f082587d97b64b5b8d0b27052604032a(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T14:56:23,180 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 25108625f673cf8c0c90412d406225a3: 2024-11-14T14:56:23,183 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C34489%2C1731596141730.1731596183183 2024-11-14T14:56:23,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:23,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:23,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:23,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:23,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:23,194 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/WALs/da1c6afcd1f9,34489,1731596141730/da1c6afcd1f9%2C34489%2C1731596141730.1731596173089 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/WALs/da1c6afcd1f9,34489,1731596141730/da1c6afcd1f9%2C34489%2C1731596141730.1731596183183 2024-11-14T14:56:23,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741839_1015 (size=2520) 2024-11-14T14:56:23,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741839_1015 (size=2520) 2024-11-14T14:56:23,204 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/WALs/da1c6afcd1f9,34489,1731596141730/da1c6afcd1f9%2C34489%2C1731596141730.1731596142117 to hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/oldWALs/da1c6afcd1f9%2C34489%2C1731596141730.1731596142117 2024-11-14T14:56:23,208 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45483:45483),(127.0.0.1/127.0.0.1:32811:32811)] 2024-11-14T14:56:23,209 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-14T14:56:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-14T14:56:23,212 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-14T14:56:23,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T14:56:23,213 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-14T14:56:23,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-14T14:56:23,332 INFO [master/da1c6afcd1f9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-14T14:56:23,332 INFO [master/da1c6afcd1f9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-14T14:56:23,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34489 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-14T14:56:23,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:56:23,367 INFO [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 25108625f673cf8c0c90412d406225a3 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-14T14:56:23,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/.tmp/info/b78cfedab50040e789a67d46b86ba071 is 1080, key is row0000/info:/1731596183182/Put/seqid=0 2024-11-14T14:56:23,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741843_1019 (size=6033) 2024-11-14T14:56:23,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741843_1019 (size=6033) 2024-11-14T14:56:23,377 INFO [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/.tmp/info/b78cfedab50040e789a67d46b86ba071 2024-11-14T14:56:23,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/.tmp/info/b78cfedab50040e789a67d46b86ba071 as hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/b78cfedab50040e789a67d46b86ba071 2024-11-14T14:56:23,389 INFO [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/b78cfedab50040e789a67d46b86ba071, entries=1, sequenceid=18, filesize=5.9 K 2024-11-14T14:56:23,390 INFO [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 25108625f673cf8c0c90412d406225a3 in 24ms, sequenceid=18, compaction requested=false 2024-11-14T14:56:23,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 25108625f673cf8c0c90412d406225a3: 2024-11-14T14:56:23,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:56:23,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-14T14:56:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-14T14:56:23,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-14T14:56:23,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 179 msec 2024-11-14T14:56:23,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 187 msec 2024-11-14T14:56:23,561 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:23,563 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:24,561 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:24,564 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:25,562 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:25,564 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:26,563 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:26,565 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:27,563 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:27,565 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:28,110 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 25108625f673cf8c0c90412d406225a3, had cached 0 bytes from a total of 14329 2024-11-14T14:56:28,564 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:28,566 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:29,564 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:29,566 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:30,565 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:30,567 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:31,566 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:31,567 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:32,566 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:32,568 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:33,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40583 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-14T14:56:33,227 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-14T14:56:33,230 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C34489%2C1731596141730.1731596193230 2024-11-14T14:56:33,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,238 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/WALs/da1c6afcd1f9,34489,1731596141730/da1c6afcd1f9%2C34489%2C1731596141730.1731596183183 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/WALs/da1c6afcd1f9,34489,1731596141730/da1c6afcd1f9%2C34489%2C1731596141730.1731596193230 2024-11-14T14:56:33,239 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45483:45483),(127.0.0.1/127.0.0.1:32811:32811)] 2024-11-14T14:56:33,239 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/WALs/da1c6afcd1f9,34489,1731596141730/da1c6afcd1f9%2C34489%2C1731596141730.1731596183183 is not closed yet, will try archiving it next time 2024-11-14T14:56:33,239 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T14:56:33,239 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:56:33,239 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:56:33,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:56:33,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741842_1018 (size=2026) 2024-11-14T14:56:33,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:56:33,239 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T14:56:33,239 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T14:56:33,239 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=196992711, stopped=false 2024-11-14T14:56:33,240 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=da1c6afcd1f9,40583,1731596141681 2024-11-14T14:56:33,240 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/WALs/da1c6afcd1f9,34489,1731596141730/da1c6afcd1f9%2C34489%2C1731596141730.1731596173089 to hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/oldWALs/da1c6afcd1f9%2C34489%2C1731596141730.1731596173089 2024-11-14T14:56:33,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741842_1018 (size=2026) 2024-11-14T14:56:33,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:56:33,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:56:33,241 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:56:33,242 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:56:33,242 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:56:33,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:56:33,242 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'da1c6afcd1f9,34489,1731596141730' ***** 2024-11-14T14:56:33,242 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T14:56:33,242 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T14:56:33,242 INFO [RS:0;da1c6afcd1f9:34489 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T14:56:33,242 INFO [RS:0;da1c6afcd1f9:34489 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T14:56:33,242 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:56:33,242 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.HRegionServer(3091): Received CLOSE for 25108625f673cf8c0c90412d406225a3 2024-11-14T14:56:33,243 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T14:56:33,243 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.HRegionServer(959): stopping server da1c6afcd1f9,34489,1731596141730 2024-11-14T14:56:33,243 INFO [RS:0;da1c6afcd1f9:34489 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:56:33,243 INFO [RS:0;da1c6afcd1f9:34489 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;da1c6afcd1f9:34489. 2024-11-14T14:56:33,243 DEBUG [RS:0;da1c6afcd1f9:34489 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:56:33,243 DEBUG [RS:0;da1c6afcd1f9:34489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:56:33,243 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 25108625f673cf8c0c90412d406225a3, disabling compactions & flushes 2024-11-14T14:56:33,243 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:56:33,243 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T14:56:33,243 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T14:56:33,243 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:56:33,243 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T14:56:33,243 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. after waiting 0 ms 2024-11-14T14:56:33,243 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:56:33,243 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T14:56:33,243 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 25108625f673cf8c0c90412d406225a3 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-14T14:56:33,243 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-14T14:56:33,243 DEBUG [RS:0;da1c6afcd1f9:34489 {}] regionserver.HRegionServer(1325): Online Regions={25108625f673cf8c0c90412d406225a3=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3., 1588230740=hbase:meta,,1.1588230740} 2024-11-14T14:56:33,244 DEBUG [RS:0;da1c6afcd1f9:34489 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 25108625f673cf8c0c90412d406225a3 2024-11-14T14:56:33,244 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:56:33,244 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:56:33,244 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:56:33,244 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:56:33,244 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:56:33,244 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB 2024-11-14T14:56:33,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:56:33,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:56:33,245 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:56:33,248 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/.tmp/info/c6ef42f74004443a99f0e5dc85225a3c is 1080, key is row0001/info:/1731596193229/Put/seqid=0 2024-11-14T14:56:33,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741845_1021 (size=6033) 2024-11-14T14:56:33,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741845_1021 (size=6033) 2024-11-14T14:56:33,254 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/.tmp/info/c6ef42f74004443a99f0e5dc85225a3c 2024-11-14T14:56:33,263 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740/.tmp/info/480c42d9157e4ae38105b26348883365 is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3./info:regioninfo/1731596143125/Put/seqid=0 2024-11-14T14:56:33,266 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/.tmp/info/c6ef42f74004443a99f0e5dc85225a3c as hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/c6ef42f74004443a99f0e5dc85225a3c 2024-11-14T14:56:33,272 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/c6ef42f74004443a99f0e5dc85225a3c, entries=1, sequenceid=22, filesize=5.9 K 2024-11-14T14:56:33,273 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 25108625f673cf8c0c90412d406225a3 in 30ms, sequenceid=22, compaction requested=true 2024-11-14T14:56:33,276 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/1a7182e7f832450686d08a824e8297ae, hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/ef8e0612a642481abb2d479a284f04b3, hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/a3b99ec727774299b09c69fb66fa0803] to archive 2024-11-14T14:56:33,277 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T14:56:33,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741846_1022 (size=7308) 2024-11-14T14:56:33,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741846_1022 (size=7308) 2024-11-14T14:56:33,279 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740/.tmp/info/480c42d9157e4ae38105b26348883365 2024-11-14T14:56:33,280 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/1a7182e7f832450686d08a824e8297ae to hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/1a7182e7f832450686d08a824e8297ae 2024-11-14T14:56:33,281 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/ef8e0612a642481abb2d479a284f04b3 to hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/ef8e0612a642481abb2d479a284f04b3 2024-11-14T14:56:33,283 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/a3b99ec727774299b09c69fb66fa0803 to hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/info/a3b99ec727774299b09c69fb66fa0803 2024-11-14T14:56:33,283 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=da1c6afcd1f9:40583 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 16 more 2024-11-14T14:56:33,284 WARN [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [1a7182e7f832450686d08a824e8297ae=6033, ef8e0612a642481abb2d479a284f04b3=6033, a3b99ec727774299b09c69fb66fa0803=6033] 2024-11-14T14:56:33,288 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/25108625f673cf8c0c90412d406225a3/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1 2024-11-14T14:56:33,289 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:56:33,289 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 25108625f673cf8c0c90412d406225a3: Waiting for close lock at 1731596193243Running coprocessor pre-close hooks at 1731596193243Disabling compacts and flushes for region at 1731596193243Disabling writes for close at 1731596193243Obtaining lock to block concurrent updates at 1731596193243Preparing flush snapshotting stores in 25108625f673cf8c0c90412d406225a3 at 1731596193243Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1731596193244 (+1 ms)Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. at 1731596193244Flushing 25108625f673cf8c0c90412d406225a3/info: creating writer at 1731596193244Flushing 25108625f673cf8c0c90412d406225a3/info: appending metadata at 1731596193248 (+4 ms)Flushing 25108625f673cf8c0c90412d406225a3/info: closing flushed file at 1731596193248Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67fc0570: reopening flushed file at 1731596193265 (+17 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 25108625f673cf8c0c90412d406225a3 in 30ms, sequenceid=22, compaction requested=true at 1731596193273 (+8 ms)Writing region close event to WAL at 1731596193284 (+11 ms)Running coprocessor post-close hooks at 1731596193289 (+5 ms)Closed at 1731596193289 2024-11-14T14:56:33,289 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731596142762.25108625f673cf8c0c90412d406225a3. 2024-11-14T14:56:33,302 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740/.tmp/ns/f2f87101a00b4d42800bcec76f40c9e7 is 43, key is default/ns:d/1731596142723/Put/seqid=0 2024-11-14T14:56:33,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741847_1023 (size=5153) 2024-11-14T14:56:33,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741847_1023 (size=5153) 2024-11-14T14:56:33,307 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740/.tmp/ns/f2f87101a00b4d42800bcec76f40c9e7 2024-11-14T14:56:33,327 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740/.tmp/table/ac898503455e42ccb32920c0c25aaf8a is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1731596143137/Put/seqid=0 2024-11-14T14:56:33,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741848_1024 (size=5508) 2024-11-14T14:56:33,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741848_1024 (size=5508) 2024-11-14T14:56:33,333 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740/.tmp/table/ac898503455e42ccb32920c0c25aaf8a 2024-11-14T14:56:33,339 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740/.tmp/info/480c42d9157e4ae38105b26348883365 as hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740/info/480c42d9157e4ae38105b26348883365 2024-11-14T14:56:33,344 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740/info/480c42d9157e4ae38105b26348883365, entries=10, sequenceid=11, filesize=7.1 K 2024-11-14T14:56:33,345 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740/.tmp/ns/f2f87101a00b4d42800bcec76f40c9e7 as hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740/ns/f2f87101a00b4d42800bcec76f40c9e7 2024-11-14T14:56:33,350 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740/ns/f2f87101a00b4d42800bcec76f40c9e7, entries=2, sequenceid=11, filesize=5.0 K 2024-11-14T14:56:33,351 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740/.tmp/table/ac898503455e42ccb32920c0c25aaf8a as hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740/table/ac898503455e42ccb32920c0c25aaf8a 2024-11-14T14:56:33,357 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740/table/ac898503455e42ccb32920c0c25aaf8a, entries=2, sequenceid=11, filesize=5.4 K 2024-11-14T14:56:33,358 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 114ms, sequenceid=11, compaction requested=false 2024-11-14T14:56:33,363 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-14T14:56:33,363 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:56:33,363 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:56:33,364 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731596193244Running coprocessor pre-close hooks at 1731596193244Disabling compacts and flushes for region at 1731596193244Disabling writes for close at 1731596193244Obtaining lock to block concurrent updates at 1731596193244Preparing flush snapshotting stores in 1588230740 at 1731596193244Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1731596193244Flushing stores of hbase:meta,,1.1588230740 at 1731596193245 (+1 ms)Flushing 1588230740/info: creating writer at 1731596193245Flushing 1588230740/info: appending metadata at 1731596193262 (+17 ms)Flushing 1588230740/info: closing flushed file at 1731596193262Flushing 1588230740/ns: creating writer at 1731596193285 (+23 ms)Flushing 1588230740/ns: appending metadata at 1731596193301 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731596193301Flushing 1588230740/table: creating writer at 1731596193312 (+11 ms)Flushing 1588230740/table: appending metadata at 1731596193326 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731596193326Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40ebe192: reopening flushed file at 1731596193338 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e72100b: reopening flushed file at 1731596193344 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@798bd349: reopening flushed file at 1731596193350 (+6 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 114ms, sequenceid=11, compaction requested=false at 1731596193358 (+8 ms)Writing region close event to WAL at 1731596193359 (+1 ms)Running coprocessor post-close hooks at 1731596193363 (+4 ms)Closed at 1731596193363 2024-11-14T14:56:33,364 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T14:56:33,444 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.HRegionServer(976): stopping server da1c6afcd1f9,34489,1731596141730; all regions closed. 2024-11-14T14:56:33,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741834_1010 (size=3306) 2024-11-14T14:56:33,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741834_1010 (size=3306) 2024-11-14T14:56:33,450 DEBUG [RS:0;da1c6afcd1f9:34489 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/oldWALs 2024-11-14T14:56:33,450 INFO [RS:0;da1c6afcd1f9:34489 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog da1c6afcd1f9%2C34489%2C1731596141730.meta:.meta(num 1731596142680) 2024-11-14T14:56:33,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741844_1020 (size=1252) 2024-11-14T14:56:33,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741844_1020 (size=1252) 2024-11-14T14:56:33,456 DEBUG [RS:0;da1c6afcd1f9:34489 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/oldWALs 2024-11-14T14:56:33,456 INFO [RS:0;da1c6afcd1f9:34489 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog da1c6afcd1f9%2C34489%2C1731596141730:(num 1731596193230) 2024-11-14T14:56:33,456 DEBUG [RS:0;da1c6afcd1f9:34489 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:56:33,456 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:56:33,456 INFO [RS:0;da1c6afcd1f9:34489 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:56:33,457 INFO [RS:0;da1c6afcd1f9:34489 {}] hbase.ChoreService(370): Chore service for: regionserver/da1c6afcd1f9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T14:56:33,457 INFO [RS:0;da1c6afcd1f9:34489 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:56:33,457 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:56:33,457 INFO [RS:0;da1c6afcd1f9:34489 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34489 2024-11-14T14:56:33,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/da1c6afcd1f9,34489,1731596141730 2024-11-14T14:56:33,459 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:56:33,459 INFO [RS:0;da1c6afcd1f9:34489 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:56:33,461 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [da1c6afcd1f9,34489,1731596141730] 2024-11-14T14:56:33,462 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/da1c6afcd1f9,34489,1731596141730 already deleted, retry=false 2024-11-14T14:56:33,463 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; da1c6afcd1f9,34489,1731596141730 expired; onlineServers=0 2024-11-14T14:56:33,463 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'da1c6afcd1f9,40583,1731596141681' ***** 2024-11-14T14:56:33,463 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T14:56:33,463 INFO [M:0;da1c6afcd1f9:40583 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:56:33,463 INFO [M:0;da1c6afcd1f9:40583 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:56:33,463 DEBUG [M:0;da1c6afcd1f9:40583 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T14:56:33,463 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T14:56:33,463 DEBUG [M:0;da1c6afcd1f9:40583 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T14:56:33,463 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731596141915 {}] cleaner.HFileCleaner(306): Exit Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731596141915,5,FailOnTimeoutGroup] 2024-11-14T14:56:33,463 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731596141915 {}] cleaner.HFileCleaner(306): Exit Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731596141915,5,FailOnTimeoutGroup] 2024-11-14T14:56:33,463 INFO [M:0;da1c6afcd1f9:40583 {}] hbase.ChoreService(370): Chore service for: master/da1c6afcd1f9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T14:56:33,463 INFO [M:0;da1c6afcd1f9:40583 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:56:33,463 DEBUG [M:0;da1c6afcd1f9:40583 {}] master.HMaster(1795): Stopping service threads 2024-11-14T14:56:33,463 INFO [M:0;da1c6afcd1f9:40583 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T14:56:33,464 INFO [M:0;da1c6afcd1f9:40583 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:56:33,464 INFO [M:0;da1c6afcd1f9:40583 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:56:33,464 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T14:56:33,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T14:56:33,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:56:33,466 DEBUG [M:0;da1c6afcd1f9:40583 {}] zookeeper.ZKUtil(347): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T14:56:33,466 WARN [M:0;da1c6afcd1f9:40583 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T14:56:33,467 INFO [M:0;da1c6afcd1f9:40583 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/.lastflushedseqids 2024-11-14T14:56:33,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741849_1025 (size=130) 2024-11-14T14:56:33,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741849_1025 (size=130) 2024-11-14T14:56:33,474 INFO [M:0;da1c6afcd1f9:40583 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:56:33,474 INFO [M:0;da1c6afcd1f9:40583 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T14:56:33,474 DEBUG [M:0;da1c6afcd1f9:40583 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:56:33,474 INFO [M:0;da1c6afcd1f9:40583 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:56:33,474 DEBUG [M:0;da1c6afcd1f9:40583 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:56:33,474 DEBUG [M:0;da1c6afcd1f9:40583 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:56:33,474 DEBUG [M:0;da1c6afcd1f9:40583 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:56:33,474 INFO [M:0;da1c6afcd1f9:40583 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.62 KB heapSize=55.02 KB 2024-11-14T14:56:33,492 DEBUG [M:0;da1c6afcd1f9:40583 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ce90d600fe6d494d99fd7ac75ef257f8 is 82, key is hbase:meta,,1/info:regioninfo/1731596142706/Put/seqid=0 2024-11-14T14:56:33,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741850_1026 (size=5672) 2024-11-14T14:56:33,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741850_1026 (size=5672) 2024-11-14T14:56:33,498 INFO [M:0;da1c6afcd1f9:40583 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ce90d600fe6d494d99fd7ac75ef257f8 2024-11-14T14:56:33,521 DEBUG [M:0;da1c6afcd1f9:40583 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/85d31e6d42ea432aa7cd3ad11314b514 is 799, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731596143141/Put/seqid=0 2024-11-14T14:56:33,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741851_1027 (size=7826) 2024-11-14T14:56:33,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741851_1027 (size=7826) 2024-11-14T14:56:33,527 INFO [M:0;da1c6afcd1f9:40583 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.02 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/85d31e6d42ea432aa7cd3ad11314b514 2024-11-14T14:56:33,532 INFO [M:0;da1c6afcd1f9:40583 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 85d31e6d42ea432aa7cd3ad11314b514 2024-11-14T14:56:33,548 DEBUG [M:0;da1c6afcd1f9:40583 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e65bbf7bbde74bb396e7d24d32423590 is 69, key is da1c6afcd1f9,34489,1731596141730/rs:state/1731596141967/Put/seqid=0 2024-11-14T14:56:33,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741852_1028 (size=5156) 2024-11-14T14:56:33,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741852_1028 (size=5156) 2024-11-14T14:56:33,553 INFO [M:0;da1c6afcd1f9:40583 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e65bbf7bbde74bb396e7d24d32423590 2024-11-14T14:56:33,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:56:33,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34489-0x101a747b9c50001, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:56:33,561 INFO [RS:0;da1c6afcd1f9:34489 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:56:33,561 INFO [RS:0;da1c6afcd1f9:34489 {}] regionserver.HRegionServer(1031): Exiting; stopping=da1c6afcd1f9,34489,1731596141730; zookeeper connection closed. 2024-11-14T14:56:33,561 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@675fef35 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@675fef35 2024-11-14T14:56:33,561 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T14:56:33,567 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:33,568 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:33,575 DEBUG [M:0;da1c6afcd1f9:40583 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b38adc20e9aa481098cff5bbcae60bdc is 52, key is load_balancer_on/state:d/1731596142758/Put/seqid=0 2024-11-14T14:56:33,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741853_1029 (size=5056) 2024-11-14T14:56:33,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741853_1029 (size=5056) 2024-11-14T14:56:33,580 INFO [M:0;da1c6afcd1f9:40583 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b38adc20e9aa481098cff5bbcae60bdc 2024-11-14T14:56:33,586 DEBUG [M:0;da1c6afcd1f9:40583 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ce90d600fe6d494d99fd7ac75ef257f8 as hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ce90d600fe6d494d99fd7ac75ef257f8 2024-11-14T14:56:33,591 INFO [M:0;da1c6afcd1f9:40583 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ce90d600fe6d494d99fd7ac75ef257f8, entries=8, sequenceid=121, filesize=5.5 K 2024-11-14T14:56:33,592 DEBUG [M:0;da1c6afcd1f9:40583 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/85d31e6d42ea432aa7cd3ad11314b514 as hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/85d31e6d42ea432aa7cd3ad11314b514 2024-11-14T14:56:33,597 INFO [M:0;da1c6afcd1f9:40583 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 85d31e6d42ea432aa7cd3ad11314b514 2024-11-14T14:56:33,597 INFO [M:0;da1c6afcd1f9:40583 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/85d31e6d42ea432aa7cd3ad11314b514, entries=14, sequenceid=121, filesize=7.6 K 2024-11-14T14:56:33,598 DEBUG [M:0;da1c6afcd1f9:40583 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e65bbf7bbde74bb396e7d24d32423590 as hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e65bbf7bbde74bb396e7d24d32423590 2024-11-14T14:56:33,603 INFO [M:0;da1c6afcd1f9:40583 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e65bbf7bbde74bb396e7d24d32423590, entries=1, sequenceid=121, filesize=5.0 K 2024-11-14T14:56:33,604 DEBUG [M:0;da1c6afcd1f9:40583 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b38adc20e9aa481098cff5bbcae60bdc as hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b38adc20e9aa481098cff5bbcae60bdc 2024-11-14T14:56:33,609 INFO [M:0;da1c6afcd1f9:40583 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37087/user/jenkins/test-data/d287ea79-88d4-f3ad-73ce-eee4e7ec09d8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b38adc20e9aa481098cff5bbcae60bdc, entries=1, sequenceid=121, filesize=4.9 K 2024-11-14T14:56:33,610 INFO [M:0;da1c6afcd1f9:40583 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.62 KB/44668, heapSize ~54.96 KB/56280, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=121, compaction requested=false 2024-11-14T14:56:33,611 INFO [M:0;da1c6afcd1f9:40583 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:56:33,611 DEBUG [M:0;da1c6afcd1f9:40583 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731596193474Disabling compacts and flushes for region at 1731596193474Disabling writes for close at 1731596193474Obtaining lock to block concurrent updates at 1731596193474Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731596193474Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44668, getHeapSize=56280, getOffHeapSize=0, getCellsCount=140 at 1731596193475 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731596193475Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731596193476 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731596193492 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731596193492Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731596193503 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731596193521 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731596193521Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731596193532 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731596193547 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731596193548 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731596193558 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731596193574 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731596193574Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11d3f23a: reopening flushed file at 1731596193586 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d19e5c8: reopening flushed file at 1731596193592 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ad3ebfa: reopening flushed file at 1731596193598 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1069a3c9: reopening flushed file at 1731596193603 (+5 ms)Finished flush of dataSize ~43.62 KB/44668, heapSize ~54.96 KB/56280, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=121, compaction requested=false at 1731596193610 (+7 ms)Writing region close event to WAL at 1731596193611 (+1 ms)Closed at 1731596193611 2024-11-14T14:56:33,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:56:33,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46057 is added to blk_1073741830_1006 (size=53065) 2024-11-14T14:56:33,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39745 is added to blk_1073741830_1006 (size=53065) 2024-11-14T14:56:33,615 INFO [M:0;da1c6afcd1f9:40583 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T14:56:33,615 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:56:33,615 INFO [M:0;da1c6afcd1f9:40583 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40583 2024-11-14T14:56:33,615 INFO [M:0;da1c6afcd1f9:40583 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:56:33,718 INFO [M:0;da1c6afcd1f9:40583 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:56:33,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:56:33,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40583-0x101a747b9c50000, quorum=127.0.0.1:57943, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:56:33,720 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@16a85bb6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:56:33,721 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5cac9c50{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:56:33,721 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:56:33,721 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4ed77c81{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:56:33,721 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6c243e85{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/hadoop.log.dir/,STOPPED} 2024-11-14T14:56:33,723 WARN [BP-309921800-172.17.0.2-1731596140861 heartbeating to localhost/127.0.0.1:37087 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:56:33,723 WARN [BP-309921800-172.17.0.2-1731596140861 heartbeating to localhost/127.0.0.1:37087 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-309921800-172.17.0.2-1731596140861 (Datanode Uuid ba51a9d6-cf30-4d31-81ba-36a08a0dd761) service to localhost/127.0.0.1:37087 2024-11-14T14:56:33,723 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:56:33,723 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:56:33,723 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/cluster_18a57b98-5841-1b8f-fcd3-15b2160d9bd5/data/data3/current/BP-309921800-172.17.0.2-1731596140861 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:56:33,724 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/cluster_18a57b98-5841-1b8f-fcd3-15b2160d9bd5/data/data4/current/BP-309921800-172.17.0.2-1731596140861 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:56:33,724 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:56:33,726 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@510b5e02{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:56:33,726 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@59a5f2c5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:56:33,726 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:56:33,727 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@168edd0a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:56:33,727 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a560185{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/hadoop.log.dir/,STOPPED} 2024-11-14T14:56:33,728 WARN [BP-309921800-172.17.0.2-1731596140861 heartbeating to localhost/127.0.0.1:37087 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:56:33,728 WARN [BP-309921800-172.17.0.2-1731596140861 heartbeating to localhost/127.0.0.1:37087 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-309921800-172.17.0.2-1731596140861 (Datanode Uuid b10d73d4-de38-4304-9c3e-a50692aba180) service to localhost/127.0.0.1:37087 2024-11-14T14:56:33,728 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:56:33,728 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:56:33,729 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/cluster_18a57b98-5841-1b8f-fcd3-15b2160d9bd5/data/data1/current/BP-309921800-172.17.0.2-1731596140861 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:56:33,729 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/cluster_18a57b98-5841-1b8f-fcd3-15b2160d9bd5/data/data2/current/BP-309921800-172.17.0.2-1731596140861 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:56:33,730 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:56:33,737 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3ff21aec{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T14:56:33,738 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@40e0483f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:56:33,738 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:56:33,738 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ece2bc8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:56:33,738 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@778c1ce5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/hadoop.log.dir/,STOPPED} 2024-11-14T14:56:33,745 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T14:56:33,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-14T14:56:33,775 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=208 (was 182) Potentially hanging thread: nioEventLoopGroup-34-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:37087 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37087 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37087 from jenkins.hfs.5 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37087 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-35-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37087 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37087 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:37087 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-35-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/da1c6afcd1f9:0.leaseChecker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82) Potentially hanging thread: HMaster-EventLoopGroup-12-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:37087 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-35-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=483 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=180 (was 167) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7252 (was 7448) 2024-11-14T14:56:33,784 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=208, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=180, ProcessCount=11, AvailableMemoryMB=7252 2024-11-14T14:56:33,784 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T14:56:33,784 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/hadoop.log.dir so I do NOT create it in target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4 2024-11-14T14:56:33,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d38dedfa-c88a-18c1-17c4-0d902dedb754/hadoop.tmp.dir so I do NOT create it in target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4 2024-11-14T14:56:33,785 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/cluster_82567f71-e0c9-9c1a-8289-309cd8e137b7, deleteOnExit=true 2024-11-14T14:56:33,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T14:56:33,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/test.cache.data in system properties and HBase conf 2024-11-14T14:56:33,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T14:56:33,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/hadoop.log.dir in system properties and HBase conf 2024-11-14T14:56:33,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T14:56:33,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T14:56:33,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T14:56:33,785 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T14:56:33,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T14:56:33,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T14:56:33,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T14:56:33,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T14:56:33,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T14:56:33,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T14:56:33,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T14:56:33,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T14:56:33,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T14:56:33,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/nfs.dump.dir in system properties and HBase conf 2024-11-14T14:56:33,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/java.io.tmpdir in system properties and HBase conf 2024-11-14T14:56:33,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T14:56:33,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T14:56:33,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T14:56:33,799 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-14T14:56:33,860 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:56:33,865 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:56:33,866 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:56:33,866 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:56:33,866 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T14:56:33,867 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:56:33,868 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@110e33ce{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:56:33,868 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52d230c9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:56:33,983 INFO [regionserver/da1c6afcd1f9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:56:33,988 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6e8e6cbc{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/java.io.tmpdir/jetty-localhost-40841-hadoop-hdfs-3_4_1-tests_jar-_-any-1510468033852658785/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T14:56:33,988 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@188ddc10{HTTP/1.1, (http/1.1)}{localhost:40841} 2024-11-14T14:56:33,989 INFO [Time-limited test {}] server.Server(415): Started @239610ms 2024-11-14T14:56:34,002 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-14T14:56:34,151 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:56:34,154 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:56:34,155 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:56:34,155 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:56:34,155 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:56:34,155 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@201bdbf9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:56:34,156 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6f8f17a1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:56:34,286 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4a67ff9c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/java.io.tmpdir/jetty-localhost-40355-hadoop-hdfs-3_4_1-tests_jar-_-any-14931446601219792358/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:56:34,286 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3bd9a438{HTTP/1.1, (http/1.1)}{localhost:40355} 2024-11-14T14:56:34,287 INFO [Time-limited test {}] server.Server(415): Started @239908ms 2024-11-14T14:56:34,288 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:56:34,323 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:56:34,326 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:56:34,327 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:56:34,327 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:56:34,327 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:56:34,327 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16ccf5f4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:56:34,328 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2735da07{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:56:34,386 WARN [Thread-1965 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/cluster_82567f71-e0c9-9c1a-8289-309cd8e137b7/data/data1/current/BP-906651502-172.17.0.2-1731596193805/current, will proceed with Du for space computation calculation, 2024-11-14T14:56:34,387 WARN [Thread-1966 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/cluster_82567f71-e0c9-9c1a-8289-309cd8e137b7/data/data2/current/BP-906651502-172.17.0.2-1731596193805/current, will proceed with Du for space computation calculation, 2024-11-14T14:56:34,410 WARN [Thread-1944 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:56:34,414 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf82b1368a2e0442f with lease ID 0x4d4649a51a9d722c: Processing first storage report for DS-e7a61dc6-83ec-4c31-a43c-3b1840ab9400 from datanode DatanodeRegistration(127.0.0.1:33753, datanodeUuid=d5faaece-58bb-47a3-8d08-d5165224d771, infoPort=34727, infoSecurePort=0, ipcPort=36085, storageInfo=lv=-57;cid=testClusterID;nsid=720680439;c=1731596193805) 2024-11-14T14:56:34,414 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf82b1368a2e0442f with lease ID 0x4d4649a51a9d722c: from storage DS-e7a61dc6-83ec-4c31-a43c-3b1840ab9400 node DatanodeRegistration(127.0.0.1:33753, datanodeUuid=d5faaece-58bb-47a3-8d08-d5165224d771, infoPort=34727, infoSecurePort=0, ipcPort=36085, storageInfo=lv=-57;cid=testClusterID;nsid=720680439;c=1731596193805), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:56:34,414 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf82b1368a2e0442f with lease ID 0x4d4649a51a9d722c: Processing first storage report for DS-9076e850-2787-49c6-ae2a-7f5ee17fc835 from datanode DatanodeRegistration(127.0.0.1:33753, datanodeUuid=d5faaece-58bb-47a3-8d08-d5165224d771, infoPort=34727, infoSecurePort=0, ipcPort=36085, storageInfo=lv=-57;cid=testClusterID;nsid=720680439;c=1731596193805) 2024-11-14T14:56:34,414 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf82b1368a2e0442f with lease ID 0x4d4649a51a9d722c: from storage DS-9076e850-2787-49c6-ae2a-7f5ee17fc835 node DatanodeRegistration(127.0.0.1:33753, datanodeUuid=d5faaece-58bb-47a3-8d08-d5165224d771, infoPort=34727, infoSecurePort=0, ipcPort=36085, storageInfo=lv=-57;cid=testClusterID;nsid=720680439;c=1731596193805), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:56:34,450 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2f932cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/java.io.tmpdir/jetty-localhost-41855-hadoop-hdfs-3_4_1-tests_jar-_-any-14661470626627054428/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:56:34,450 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5ff7780b{HTTP/1.1, (http/1.1)}{localhost:41855} 2024-11-14T14:56:34,450 INFO [Time-limited test {}] server.Server(415): Started @240072ms 2024-11-14T14:56:34,451 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:56:34,549 WARN [Thread-1991 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/cluster_82567f71-e0c9-9c1a-8289-309cd8e137b7/data/data3/current/BP-906651502-172.17.0.2-1731596193805/current, will proceed with Du for space computation calculation, 2024-11-14T14:56:34,549 WARN [Thread-1992 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/cluster_82567f71-e0c9-9c1a-8289-309cd8e137b7/data/data4/current/BP-906651502-172.17.0.2-1731596193805/current, will proceed with Du for space computation calculation, 2024-11-14T14:56:34,566 WARN [Thread-1980 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:56:34,567 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:34,568 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb4615917d39440c1 with lease ID 0x4d4649a51a9d722d: Processing first storage report for DS-5bc881ab-80b1-41f4-b3bc-0587808bc15c from datanode DatanodeRegistration(127.0.0.1:35037, datanodeUuid=df7bd351-504e-49a5-9f21-62de66989bf3, infoPort=43735, infoSecurePort=0, ipcPort=43083, storageInfo=lv=-57;cid=testClusterID;nsid=720680439;c=1731596193805) 2024-11-14T14:56:34,568 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb4615917d39440c1 with lease ID 0x4d4649a51a9d722d: from storage DS-5bc881ab-80b1-41f4-b3bc-0587808bc15c node DatanodeRegistration(127.0.0.1:35037, datanodeUuid=df7bd351-504e-49a5-9f21-62de66989bf3, infoPort=43735, infoSecurePort=0, ipcPort=43083, storageInfo=lv=-57;cid=testClusterID;nsid=720680439;c=1731596193805), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:56:34,568 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb4615917d39440c1 with lease ID 0x4d4649a51a9d722d: Processing first storage report for DS-bd27da3d-d6a1-4a2d-809b-797dd5f97859 from datanode DatanodeRegistration(127.0.0.1:35037, datanodeUuid=df7bd351-504e-49a5-9f21-62de66989bf3, infoPort=43735, infoSecurePort=0, ipcPort=43083, storageInfo=lv=-57;cid=testClusterID;nsid=720680439;c=1731596193805) 2024-11-14T14:56:34,569 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb4615917d39440c1 with lease ID 0x4d4649a51a9d722d: from storage DS-bd27da3d-d6a1-4a2d-809b-797dd5f97859 node DatanodeRegistration(127.0.0.1:35037, datanodeUuid=df7bd351-504e-49a5-9f21-62de66989bf3, infoPort=43735, infoSecurePort=0, ipcPort=43083, storageInfo=lv=-57;cid=testClusterID;nsid=720680439;c=1731596193805), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:56:34,568 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:34,575 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4 2024-11-14T14:56:34,579 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/cluster_82567f71-e0c9-9c1a-8289-309cd8e137b7/zookeeper_0, clientPort=63286, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/cluster_82567f71-e0c9-9c1a-8289-309cd8e137b7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/cluster_82567f71-e0c9-9c1a-8289-309cd8e137b7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T14:56:34,580 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63286 2024-11-14T14:56:34,580 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:56:34,582 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:56:34,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:56:34,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:56:34,592 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0 with version=8 2024-11-14T14:56:34,592 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/hbase-staging 2024-11-14T14:56:34,593 INFO [Time-limited test {}] client.ConnectionUtils(128): master/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:56:34,594 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:56:34,594 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:56:34,594 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:56:34,594 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:56:34,594 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:56:34,594 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T14:56:34,594 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:56:34,594 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41179 2024-11-14T14:56:34,595 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41179 connecting to ZooKeeper ensemble=127.0.0.1:63286 2024-11-14T14:56:34,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:411790x0, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:56:34,602 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41179-0x101a748887a0000 connected 2024-11-14T14:56:34,619 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:56:34,620 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:56:34,623 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:56:34,623 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0, hbase.cluster.distributed=false 2024-11-14T14:56:34,625 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:56:34,626 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41179 2024-11-14T14:56:34,626 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41179 2024-11-14T14:56:34,629 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41179 2024-11-14T14:56:34,629 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41179 2024-11-14T14:56:34,629 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41179 2024-11-14T14:56:34,645 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:56:34,645 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:56:34,645 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:56:34,645 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:56:34,645 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:56:34,645 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:56:34,645 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T14:56:34,646 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:56:34,646 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36857 2024-11-14T14:56:34,648 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36857 connecting to ZooKeeper ensemble=127.0.0.1:63286 2024-11-14T14:56:34,648 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:56:34,650 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:56:34,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:368570x0, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:56:34,655 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:368570x0, quorum=127.0.0.1:63286, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:56:34,655 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36857-0x101a748887a0001 connected 2024-11-14T14:56:34,655 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T14:56:34,656 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T14:56:34,656 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T14:56:34,657 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:56:34,658 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36857 2024-11-14T14:56:34,658 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36857 2024-11-14T14:56:34,658 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36857 2024-11-14T14:56:34,659 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36857 2024-11-14T14:56:34,659 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36857 2024-11-14T14:56:34,671 DEBUG [M:0;da1c6afcd1f9:41179 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;da1c6afcd1f9:41179 2024-11-14T14:56:34,671 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/da1c6afcd1f9,41179,1731596194593 2024-11-14T14:56:34,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:56:34,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:56:34,673 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/da1c6afcd1f9,41179,1731596194593 2024-11-14T14:56:34,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T14:56:34,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:56:34,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:56:34,675 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T14:56:34,676 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/da1c6afcd1f9,41179,1731596194593 from backup master directory 2024-11-14T14:56:34,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/da1c6afcd1f9,41179,1731596194593 2024-11-14T14:56:34,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:56:34,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:56:34,678 WARN [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:56:34,678 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=da1c6afcd1f9,41179,1731596194593 2024-11-14T14:56:34,682 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/hbase.id] with ID: 95afef83-7d49-413a-8a06-fac6d1485b2a 2024-11-14T14:56:34,682 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/.tmp/hbase.id 2024-11-14T14:56:34,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:56:34,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:56:34,689 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/.tmp/hbase.id]:[hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/hbase.id] 2024-11-14T14:56:34,702 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:56:34,703 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T14:56:34,704 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-14T14:56:34,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:56:34,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:56:34,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:56:34,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:56:34,715 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T14:56:34,715 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T14:56:34,716 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:56:34,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:56:34,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:56:34,723 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store 2024-11-14T14:56:34,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:56:34,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:56:34,731 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:56:34,731 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:56:34,731 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:56:34,731 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:56:34,731 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:56:34,731 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:56:34,731 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:56:34,731 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731596194731Disabling compacts and flushes for region at 1731596194731Disabling writes for close at 1731596194731Writing region close event to WAL at 1731596194731Closed at 1731596194731 2024-11-14T14:56:34,732 WARN [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/.initializing 2024-11-14T14:56:34,732 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/WALs/da1c6afcd1f9,41179,1731596194593 2024-11-14T14:56:34,735 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C41179%2C1731596194593, suffix=, logDir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/WALs/da1c6afcd1f9,41179,1731596194593, archiveDir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/oldWALs, maxLogs=10 2024-11-14T14:56:34,735 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C41179%2C1731596194593.1731596194735 2024-11-14T14:56:34,740 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/WALs/da1c6afcd1f9,41179,1731596194593/da1c6afcd1f9%2C41179%2C1731596194593.1731596194735 2024-11-14T14:56:34,740 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43735:43735),(127.0.0.1/127.0.0.1:34727:34727)] 2024-11-14T14:56:34,741 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:56:34,741 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:56:34,741 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:56:34,741 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:56:34,743 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:56:34,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T14:56:34,744 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:34,745 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:56:34,745 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:56:34,746 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T14:56:34,746 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:34,746 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:56:34,746 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:56:34,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T14:56:34,748 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:34,748 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:56:34,748 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:56:34,749 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T14:56:34,749 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:34,750 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:56:34,750 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:56:34,750 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:56:34,751 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:56:34,752 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:56:34,752 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:56:34,753 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T14:56:34,754 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:56:34,756 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:56:34,757 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=695736, jitterRate=-0.1153271347284317}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T14:56:34,757 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731596194742Initializing all the Stores at 1731596194742Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596194742Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596194743 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596194743Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596194743Cleaning up temporary data from old regions at 1731596194752 (+9 ms)Region opened successfully at 1731596194757 (+5 ms) 2024-11-14T14:56:34,758 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T14:56:34,760 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236f9c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:56:34,761 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T14:56:34,761 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T14:56:34,761 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T14:56:34,762 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:56:34,762 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:56:34,762 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-14T14:56:34,762 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T14:56:34,764 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T14:56:34,765 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T14:56:34,767 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T14:56:34,767 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T14:56:34,768 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T14:56:34,770 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T14:56:34,770 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T14:56:34,771 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T14:56:34,773 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T14:56:34,774 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T14:56:34,776 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T14:56:34,778 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T14:56:34,780 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T14:56:34,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:56:34,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:56:34,782 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=da1c6afcd1f9,41179,1731596194593, sessionid=0x101a748887a0000, setting cluster-up flag (Was=false) 2024-11-14T14:56:34,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:56:34,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:56:34,786 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:56:34,791 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T14:56:34,792 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da1c6afcd1f9,41179,1731596194593 2024-11-14T14:56:34,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:56:34,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:56:34,801 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T14:56:34,802 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da1c6afcd1f9,41179,1731596194593 2024-11-14T14:56:34,803 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T14:56:34,805 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:56:34,805 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T14:56:34,805 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T14:56:34,805 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: da1c6afcd1f9,41179,1731596194593 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T14:56:34,806 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:56:34,807 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:56:34,807 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:56:34,807 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:56:34,807 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/da1c6afcd1f9:0, corePoolSize=10, maxPoolSize=10 2024-11-14T14:56:34,807 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:56:34,807 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:56:34,807 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:56:34,808 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731596224808 2024-11-14T14:56:34,809 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T14:56:34,809 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:56:34,809 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T14:56:34,809 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T14:56:34,809 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T14:56:34,809 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T14:56:34,809 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T14:56:34,809 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T14:56:34,809 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:34,809 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T14:56:34,809 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T14:56:34,809 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T14:56:34,810 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:34,810 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T14:56:34,810 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T14:56:34,810 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T14:56:34,810 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731596194810,5,FailOnTimeoutGroup] 2024-11-14T14:56:34,810 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731596194810,5,FailOnTimeoutGroup] 2024-11-14T14:56:34,810 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:34,810 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T14:56:34,810 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:34,810 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:34,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:56:34,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:56:34,816 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T14:56:34,817 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0 2024-11-14T14:56:34,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:56:34,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:56:34,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:56:34,824 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:56:34,825 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:56:34,826 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:34,826 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:56:34,826 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:56:34,827 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:56:34,827 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:34,828 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:56:34,828 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:56:34,829 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:56:34,829 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:34,829 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:56:34,830 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:56:34,831 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:56:34,831 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:34,831 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:56:34,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:56:34,832 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740 2024-11-14T14:56:34,832 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740 2024-11-14T14:56:34,834 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:56:34,834 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:56:34,834 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T14:56:34,835 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:56:34,837 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:56:34,838 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=758848, jitterRate=-0.03507542610168457}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T14:56:34,839 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731596194823Initializing all the Stores at 1731596194824 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596194824Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596194824Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596194824Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596194824Cleaning up temporary data from old regions at 1731596194834 (+10 ms)Region opened successfully at 1731596194839 (+5 ms) 2024-11-14T14:56:34,839 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:56:34,839 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:56:34,839 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:56:34,839 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:56:34,839 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:56:34,839 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:56:34,839 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731596194839Disabling compacts and flushes for region at 1731596194839Disabling writes for close at 1731596194839Writing region close event to WAL at 1731596194839Closed at 1731596194839 2024-11-14T14:56:34,841 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:56:34,841 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T14:56:34,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T14:56:34,842 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:56:34,843 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T14:56:34,860 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.HRegionServer(746): ClusterId : 95afef83-7d49-413a-8a06-fac6d1485b2a 2024-11-14T14:56:34,860 DEBUG [RS:0;da1c6afcd1f9:36857 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T14:56:34,863 DEBUG [RS:0;da1c6afcd1f9:36857 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T14:56:34,863 DEBUG [RS:0;da1c6afcd1f9:36857 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T14:56:34,866 DEBUG [RS:0;da1c6afcd1f9:36857 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T14:56:34,866 DEBUG [RS:0;da1c6afcd1f9:36857 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ead7513, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:56:34,878 DEBUG [RS:0;da1c6afcd1f9:36857 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;da1c6afcd1f9:36857 2024-11-14T14:56:34,878 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T14:56:34,878 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T14:56:34,878 DEBUG [RS:0;da1c6afcd1f9:36857 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T14:56:34,878 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.HRegionServer(2659): reportForDuty to master=da1c6afcd1f9,41179,1731596194593 with port=36857, startcode=1731596194645 2024-11-14T14:56:34,878 DEBUG [RS:0;da1c6afcd1f9:36857 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T14:56:34,881 INFO [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49395, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T14:56:34,881 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41179 {}] master.ServerManager(363): Checking decommissioned status of RegionServer da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:34,881 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41179 {}] master.ServerManager(517): Registering regionserver=da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:34,882 DEBUG [RS:0;da1c6afcd1f9:36857 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0 2024-11-14T14:56:34,882 DEBUG [RS:0;da1c6afcd1f9:36857 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41455 2024-11-14T14:56:34,882 DEBUG [RS:0;da1c6afcd1f9:36857 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T14:56:34,884 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:56:34,885 DEBUG [RS:0;da1c6afcd1f9:36857 {}] zookeeper.ZKUtil(111): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:34,885 WARN [RS:0;da1c6afcd1f9:36857 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:56:34,885 INFO [RS:0;da1c6afcd1f9:36857 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:56:34,885 DEBUG [RS:0;da1c6afcd1f9:36857 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/WALs/da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:34,885 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [da1c6afcd1f9,36857,1731596194645] 2024-11-14T14:56:34,893 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T14:56:34,895 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T14:56:34,895 INFO [RS:0;da1c6afcd1f9:36857 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T14:56:34,895 INFO [RS:0;da1c6afcd1f9:36857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:34,896 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T14:56:34,896 INFO [RS:0;da1c6afcd1f9:36857 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T14:56:34,897 INFO [RS:0;da1c6afcd1f9:36857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:34,897 DEBUG [RS:0;da1c6afcd1f9:36857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:56:34,897 DEBUG [RS:0;da1c6afcd1f9:36857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:56:34,897 DEBUG [RS:0;da1c6afcd1f9:36857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:56:34,897 DEBUG [RS:0;da1c6afcd1f9:36857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:56:34,897 DEBUG [RS:0;da1c6afcd1f9:36857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:56:34,897 DEBUG [RS:0;da1c6afcd1f9:36857 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:56:34,897 DEBUG [RS:0;da1c6afcd1f9:36857 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:56:34,897 DEBUG [RS:0;da1c6afcd1f9:36857 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:56:34,897 DEBUG [RS:0;da1c6afcd1f9:36857 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:56:34,897 DEBUG [RS:0;da1c6afcd1f9:36857 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:56:34,897 DEBUG [RS:0;da1c6afcd1f9:36857 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:56:34,897 DEBUG [RS:0;da1c6afcd1f9:36857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:56:34,897 DEBUG [RS:0;da1c6afcd1f9:36857 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:56:34,897 DEBUG [RS:0;da1c6afcd1f9:36857 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:56:34,898 INFO [RS:0;da1c6afcd1f9:36857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:34,898 INFO [RS:0;da1c6afcd1f9:36857 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:34,898 INFO [RS:0;da1c6afcd1f9:36857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:34,898 INFO [RS:0;da1c6afcd1f9:36857 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:34,898 INFO [RS:0;da1c6afcd1f9:36857 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:34,898 INFO [RS:0;da1c6afcd1f9:36857 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,36857,1731596194645-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:56:34,916 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T14:56:34,916 INFO [RS:0;da1c6afcd1f9:36857 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,36857,1731596194645-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:34,916 INFO [RS:0;da1c6afcd1f9:36857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:34,916 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.Replication(171): da1c6afcd1f9,36857,1731596194645 started 2024-11-14T14:56:34,932 INFO [RS:0;da1c6afcd1f9:36857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:34,932 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.HRegionServer(1482): Serving as da1c6afcd1f9,36857,1731596194645, RpcServer on da1c6afcd1f9/172.17.0.2:36857, sessionid=0x101a748887a0001 2024-11-14T14:56:34,932 DEBUG [RS:0;da1c6afcd1f9:36857 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T14:56:34,932 DEBUG [RS:0;da1c6afcd1f9:36857 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:34,932 DEBUG [RS:0;da1c6afcd1f9:36857 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,36857,1731596194645' 2024-11-14T14:56:34,932 DEBUG [RS:0;da1c6afcd1f9:36857 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T14:56:34,933 DEBUG [RS:0;da1c6afcd1f9:36857 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T14:56:34,933 DEBUG [RS:0;da1c6afcd1f9:36857 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T14:56:34,933 DEBUG [RS:0;da1c6afcd1f9:36857 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T14:56:34,933 DEBUG [RS:0;da1c6afcd1f9:36857 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:34,933 DEBUG [RS:0;da1c6afcd1f9:36857 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,36857,1731596194645' 2024-11-14T14:56:34,933 DEBUG [RS:0;da1c6afcd1f9:36857 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T14:56:34,934 DEBUG [RS:0;da1c6afcd1f9:36857 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T14:56:34,934 DEBUG [RS:0;da1c6afcd1f9:36857 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T14:56:34,934 INFO [RS:0;da1c6afcd1f9:36857 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T14:56:34,934 INFO [RS:0;da1c6afcd1f9:36857 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T14:56:34,994 WARN [da1c6afcd1f9:41179 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-14T14:56:35,036 INFO [RS:0;da1c6afcd1f9:36857 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C36857%2C1731596194645, suffix=, logDir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/WALs/da1c6afcd1f9,36857,1731596194645, archiveDir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/oldWALs, maxLogs=32 2024-11-14T14:56:35,037 INFO [RS:0;da1c6afcd1f9:36857 {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C36857%2C1731596194645.1731596195037 2024-11-14T14:56:35,043 INFO [RS:0;da1c6afcd1f9:36857 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/WALs/da1c6afcd1f9,36857,1731596194645/da1c6afcd1f9%2C36857%2C1731596194645.1731596195037 2024-11-14T14:56:35,049 DEBUG [RS:0;da1c6afcd1f9:36857 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34727:34727),(127.0.0.1/127.0.0.1:43735:43735)] 2024-11-14T14:56:35,244 DEBUG [da1c6afcd1f9:41179 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T14:56:35,244 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:35,246 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,36857,1731596194645, state=OPENING 2024-11-14T14:56:35,247 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T14:56:35,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:56:35,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:56:35,250 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:56:35,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,36857,1731596194645}] 2024-11-14T14:56:35,250 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:56:35,250 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:56:35,403 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T14:56:35,405 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36481, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T14:56:35,409 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T14:56:35,409 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:56:35,411 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C36857%2C1731596194645.meta, suffix=.meta, logDir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/WALs/da1c6afcd1f9,36857,1731596194645, archiveDir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/oldWALs, maxLogs=32 2024-11-14T14:56:35,412 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C36857%2C1731596194645.meta.1731596195412.meta 2024-11-14T14:56:35,422 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/WALs/da1c6afcd1f9,36857,1731596194645/da1c6afcd1f9%2C36857%2C1731596194645.meta.1731596195412.meta 2024-11-14T14:56:35,423 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34727:34727),(127.0.0.1/127.0.0.1:43735:43735)] 2024-11-14T14:56:35,423 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:56:35,424 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T14:56:35,424 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T14:56:35,424 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T14:56:35,424 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T14:56:35,424 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:56:35,424 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T14:56:35,424 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T14:56:35,425 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:56:35,426 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:56:35,426 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:35,427 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:56:35,427 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:56:35,428 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:56:35,428 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:35,428 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:56:35,428 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:56:35,429 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:56:35,429 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:35,429 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:56:35,430 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:56:35,430 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:56:35,430 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:35,431 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:56:35,431 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:56:35,431 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740 2024-11-14T14:56:35,432 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740 2024-11-14T14:56:35,434 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:56:35,434 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:56:35,434 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T14:56:35,436 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:56:35,437 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=780588, jitterRate=-0.007431432604789734}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T14:56:35,437 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T14:56:35,437 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731596195424Writing region info on filesystem at 1731596195424Initializing all the Stores at 1731596195425 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596195425Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596195425Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596195425Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596195425Cleaning up temporary data from old regions at 1731596195434 (+9 ms)Running coprocessor post-open hooks at 1731596195437 (+3 ms)Region opened successfully at 1731596195437 2024-11-14T14:56:35,439 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731596195403 2024-11-14T14:56:35,441 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T14:56:35,441 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T14:56:35,442 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:35,443 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,36857,1731596194645, state=OPEN 2024-11-14T14:56:35,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:56:35,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:56:35,448 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:56:35,448 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:35,448 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:56:35,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T14:56:35,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,36857,1731596194645 in 198 msec 2024-11-14T14:56:35,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T14:56:35,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 609 msec 2024-11-14T14:56:35,453 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:56:35,453 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T14:56:35,455 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:56:35,455 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,36857,1731596194645, seqNum=-1] 2024-11-14T14:56:35,455 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:56:35,456 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56685, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:56:35,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 656 msec 2024-11-14T14:56:35,462 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731596195462, completionTime=-1 2024-11-14T14:56:35,462 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T14:56:35,462 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:56:35,464 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-14T14:56:35,464 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731596255464 2024-11-14T14:56:35,464 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731596315464 2024-11-14T14:56:35,464 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-14T14:56:35,464 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,41179,1731596194593-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:35,464 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,41179,1731596194593-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:35,464 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,41179,1731596194593-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:35,464 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-da1c6afcd1f9:41179, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:35,464 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:35,465 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:35,466 DEBUG [master/da1c6afcd1f9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T14:56:35,469 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.791sec 2024-11-14T14:56:35,469 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T14:56:35,469 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T14:56:35,469 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T14:56:35,469 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T14:56:35,469 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T14:56:35,469 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,41179,1731596194593-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:56:35,469 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,41179,1731596194593-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T14:56:35,471 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T14:56:35,471 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T14:56:35,471 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,41179,1731596194593-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:56:35,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@444decb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:56:35,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request da1c6afcd1f9,41179,-1 for getting cluster id 2024-11-14T14:56:35,561 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T14:56:35,562 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '95afef83-7d49-413a-8a06-fac6d1485b2a' 2024-11-14T14:56:35,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T14:56:35,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "95afef83-7d49-413a-8a06-fac6d1485b2a" 2024-11-14T14:56:35,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17f97e06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:56:35,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [da1c6afcd1f9,41179,-1] 2024-11-14T14:56:35,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T14:56:35,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:56:35,565 INFO [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T14:56:35,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ee7ab7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:56:35,566 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:56:35,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,36857,1731596194645, seqNum=-1] 2024-11-14T14:56:35,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:56:35,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:56:35,568 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:35,569 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:35,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=da1c6afcd1f9,41179,1731596194593 2024-11-14T14:56:35,570 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:56:35,572 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-14T14:56:35,573 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T14:56:35,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is da1c6afcd1f9,41179,1731596194593 2024-11-14T14:56:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2d29d2f5 2024-11-14T14:56:35,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T14:56:35,575 INFO [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T14:56:35,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41179 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-14T14:56:35,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41179 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-14T14:56:35,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41179 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T14:56:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41179 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling 2024-11-14T14:56:35,579 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T14:56:35,579 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:35,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41179 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRolling" procId is: 4 2024-11-14T14:56:35,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41179 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:56:35,580 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T14:56:35,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741835_1011 (size=381) 2024-11-14T14:56:35,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741835_1011 (size=381) 2024-11-14T14:56:35,589 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8078e0e84354cb525a824451f0593eef, NAME => 'TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0 2024-11-14T14:56:35,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741836_1012 (size=64) 2024-11-14T14:56:35,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741836_1012 (size=64) 2024-11-14T14:56:35,596 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:56:35,596 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 8078e0e84354cb525a824451f0593eef, disabling compactions & flushes 2024-11-14T14:56:35,596 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. 2024-11-14T14:56:35,596 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. 2024-11-14T14:56:35,596 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. after waiting 0 ms 2024-11-14T14:56:35,596 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. 2024-11-14T14:56:35,596 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. 2024-11-14T14:56:35,597 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8078e0e84354cb525a824451f0593eef: Waiting for close lock at 1731596195596Disabling compacts and flushes for region at 1731596195596Disabling writes for close at 1731596195596Writing region close event to WAL at 1731596195596Closed at 1731596195596 2024-11-14T14:56:35,598 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T14:56:35,598 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731596195598"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731596195598"}]},"ts":"1731596195598"} 2024-11-14T14:56:35,601 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T14:56:35,602 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T14:56:35,602 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731596195602"}]},"ts":"1731596195602"} 2024-11-14T14:56:35,604 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta 2024-11-14T14:56:35,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8078e0e84354cb525a824451f0593eef, ASSIGN}] 2024-11-14T14:56:35,605 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8078e0e84354cb525a824451f0593eef, ASSIGN 2024-11-14T14:56:35,606 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8078e0e84354cb525a824451f0593eef, ASSIGN; state=OFFLINE, location=da1c6afcd1f9,36857,1731596194645; forceNewPlan=false, retain=false 2024-11-14T14:56:35,757 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8078e0e84354cb525a824451f0593eef, regionState=OPENING, regionLocation=da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:35,760 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8078e0e84354cb525a824451f0593eef, ASSIGN because future has completed 2024-11-14T14:56:35,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8078e0e84354cb525a824451f0593eef, server=da1c6afcd1f9,36857,1731596194645}] 2024-11-14T14:56:35,917 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. 2024-11-14T14:56:35,918 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8078e0e84354cb525a824451f0593eef, NAME => 'TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:56:35,918 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:35,918 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:56:35,918 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:35,918 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:35,919 INFO [StoreOpener-8078e0e84354cb525a824451f0593eef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:35,921 INFO [StoreOpener-8078e0e84354cb525a824451f0593eef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8078e0e84354cb525a824451f0593eef columnFamilyName info 2024-11-14T14:56:35,921 DEBUG [StoreOpener-8078e0e84354cb525a824451f0593eef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:35,921 INFO [StoreOpener-8078e0e84354cb525a824451f0593eef-1 {}] regionserver.HStore(327): Store=8078e0e84354cb525a824451f0593eef/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:56:35,921 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:35,922 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:35,922 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:35,923 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:35,923 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:35,925 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:35,927 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:56:35,927 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8078e0e84354cb525a824451f0593eef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=701956, jitterRate=-0.10741694271564484}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:56:35,928 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:35,928 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8078e0e84354cb525a824451f0593eef: Running coprocessor pre-open hook at 1731596195918Writing region info on filesystem at 1731596195918Initializing all the Stores at 1731596195919 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596195919Cleaning up temporary data from old regions at 1731596195923 (+4 ms)Running coprocessor post-open hooks at 1731596195928 (+5 ms)Region opened successfully at 1731596195928 2024-11-14T14:56:35,929 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef., pid=6, masterSystemTime=1731596195913 2024-11-14T14:56:35,932 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. 2024-11-14T14:56:35,932 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. 2024-11-14T14:56:35,933 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8078e0e84354cb525a824451f0593eef, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:35,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8078e0e84354cb525a824451f0593eef, server=da1c6afcd1f9,36857,1731596194645 because future has completed 2024-11-14T14:56:35,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T14:56:35,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8078e0e84354cb525a824451f0593eef, server=da1c6afcd1f9,36857,1731596194645 in 176 msec 2024-11-14T14:56:35,942 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T14:56:35,942 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8078e0e84354cb525a824451f0593eef, ASSIGN in 335 msec 2024-11-14T14:56:35,943 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T14:56:35,943 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731596195943"}]},"ts":"1731596195943"} 2024-11-14T14:56:35,945 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta 2024-11-14T14:56:35,946 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T14:56:35,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 370 msec 2024-11-14T14:56:36,568 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:36,569 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:37,569 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:37,570 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:38,290 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,290 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,290 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,290 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,290 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,290 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,291 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,291 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,304 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,304 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,305 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,305 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,305 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,305 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,308 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,308 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,309 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,310 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,570 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:38,570 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:38,815 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-14T14:56:38,816 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,816 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,816 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,817 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,817 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,817 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,818 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,818 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,837 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,837 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,837 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,838 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,838 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,839 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,842 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,843 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,843 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:38,846 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:39,570 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:39,571 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:39,616 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-11-14T14:56:39,617 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer 2024-11-14T14:56:39,617 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-14T14:56:40,571 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:40,571 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:40,894 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T14:56:40,894 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling' 2024-11-14T14:56:41,571 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:41,571 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:42,572 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:42,572 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:43,573 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:43,573 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:44,573 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:44,573 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:45,120 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-14T14:56:45,121 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:45,121 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:45,121 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:45,121 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:45,121 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:45,122 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:45,122 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:45,122 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:45,142 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:45,142 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:45,142 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:45,143 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:45,143 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:45,143 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:45,148 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:45,148 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:45,148 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:45,151 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:45,574 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:45,574 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41179 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T14:56:45,597 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed 2024-11-14T14:56:45,597 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100 2024-11-14T14:56:45,600 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling 2024-11-14T14:56:45,600 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. 2024-11-14T14:56:45,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef., hostname=da1c6afcd1f9,36857,1731596194645, seqNum=2] 2024-11-14T14:56:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:45,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8078e0e84354cb525a824451f0593eef 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-14T14:56:45,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/1066df47e5b345e58248fc30804d930d is 1080, key is row0001/info:/1731596205604/Put/seqid=0 2024-11-14T14:56:45,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741837_1013 (size=12509) 2024-11-14T14:56:45,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741837_1013 (size=12509) 2024-11-14T14:56:45,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/1066df47e5b345e58248fc30804d930d 2024-11-14T14:56:45,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/1066df47e5b345e58248fc30804d930d as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/1066df47e5b345e58248fc30804d930d 2024-11-14T14:56:45,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/1066df47e5b345e58248fc30804d930d, entries=7, sequenceid=11, filesize=12.2 K 2024-11-14T14:56:45,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=23.12 KB/23672 for 8078e0e84354cb525a824451f0593eef in 40ms, sequenceid=11, compaction requested=false 2024-11-14T14:56:45,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8078e0e84354cb525a824451f0593eef: 2024-11-14T14:56:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:45,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8078e0e84354cb525a824451f0593eef 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB 2024-11-14T14:56:45,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/6a80e95af09c481092630a399a665bec is 1080, key is row0008/info:/1731596205616/Put/seqid=0 2024-11-14T14:56:45,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741838_1014 (size=29761) 2024-11-14T14:56:45,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741838_1014 (size=29761) 2024-11-14T14:56:45,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/6a80e95af09c481092630a399a665bec 2024-11-14T14:56:45,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/6a80e95af09c481092630a399a665bec as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/6a80e95af09c481092630a399a665bec 2024-11-14T14:56:45,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/6a80e95af09c481092630a399a665bec, entries=23, sequenceid=37, filesize=29.1 K 2024-11-14T14:56:45,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for 8078e0e84354cb525a824451f0593eef in 23ms, sequenceid=37, compaction requested=false 2024-11-14T14:56:45,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8078e0e84354cb525a824451f0593eef: 2024-11-14T14:56:45,679 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=41.3 K, sizeToCheck=16.0 K 2024-11-14T14:56:45,679 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:56:45,679 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/6a80e95af09c481092630a399a665bec because midkey is the same as first or last row 2024-11-14T14:56:46,574 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:46,574 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:47,575 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:47,575 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:47,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8078e0e84354cb525a824451f0593eef 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-14T14:56:47,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/177bc9bf45844277af42a42f70b3bf60 is 1080, key is row0031/info:/1731596205657/Put/seqid=0 2024-11-14T14:56:47,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741839_1015 (size=12509) 2024-11-14T14:56:47,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741839_1015 (size=12509) 2024-11-14T14:56:47,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/177bc9bf45844277af42a42f70b3bf60 2024-11-14T14:56:47,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/177bc9bf45844277af42a42f70b3bf60 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/177bc9bf45844277af42a42f70b3bf60 2024-11-14T14:56:47,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/177bc9bf45844277af42a42f70b3bf60, entries=7, sequenceid=47, filesize=12.2 K 2024-11-14T14:56:47,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for 8078e0e84354cb525a824451f0593eef in 27ms, sequenceid=47, compaction requested=true 2024-11-14T14:56:47,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8078e0e84354cb525a824451f0593eef: 2024-11-14T14:56:47,697 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=53.5 K, sizeToCheck=16.0 K 2024-11-14T14:56:47,697 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:56:47,697 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/6a80e95af09c481092630a399a665bec because midkey is the same as first or last row 2024-11-14T14:56:47,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8078e0e84354cb525a824451f0593eef:info, priority=-2147483648, current under compaction store size is 1 2024-11-14T14:56:47,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:56:47,698 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T14:56:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:47,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8078e0e84354cb525a824451f0593eef 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-11-14T14:56:47,699 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 54779 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T14:56:47,699 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1541): 8078e0e84354cb525a824451f0593eef/info is initiating minor compaction (all files) 2024-11-14T14:56:47,699 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8078e0e84354cb525a824451f0593eef/info in TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. 2024-11-14T14:56:47,699 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/1066df47e5b345e58248fc30804d930d, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/6a80e95af09c481092630a399a665bec, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/177bc9bf45844277af42a42f70b3bf60] into tmpdir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp, totalSize=53.5 K 2024-11-14T14:56:47,700 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1066df47e5b345e58248fc30804d930d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731596205604 2024-11-14T14:56:47,700 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a80e95af09c481092630a399a665bec, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731596205616 2024-11-14T14:56:47,701 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 177bc9bf45844277af42a42f70b3bf60, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731596205657 2024-11-14T14:56:47,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/43c916d615d1427b9193d9f08485243a is 1080, key is row0038/info:/1731596207671/Put/seqid=0 2024-11-14T14:56:47,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741840_1016 (size=21141) 2024-11-14T14:56:47,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741840_1016 (size=21141) 2024-11-14T14:56:47,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/43c916d615d1427b9193d9f08485243a 2024-11-14T14:56:47,731 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8078e0e84354cb525a824451f0593eef#info#compaction#60 average throughput is 18.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T14:56:47,732 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/4367cc6e1de5469f9bd6aea0d297b331 is 1080, key is row0001/info:/1731596205604/Put/seqid=0 2024-11-14T14:56:47,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/43c916d615d1427b9193d9f08485243a as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/43c916d615d1427b9193d9f08485243a 2024-11-14T14:56:47,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/43c916d615d1427b9193d9f08485243a, entries=15, sequenceid=65, filesize=20.6 K 2024-11-14T14:56:47,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=12.61 KB/12912 for 8078e0e84354cb525a824451f0593eef in 47ms, sequenceid=65, compaction requested=false 2024-11-14T14:56:47,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8078e0e84354cb525a824451f0593eef: 2024-11-14T14:56:47,745 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=74.1 K, sizeToCheck=16.0 K 2024-11-14T14:56:47,745 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:56:47,745 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/6a80e95af09c481092630a399a665bec because midkey is the same as first or last row 2024-11-14T14:56:47,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741841_1017 (size=44978) 2024-11-14T14:56:47,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741841_1017 (size=44978) 2024-11-14T14:56:47,759 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/4367cc6e1de5469f9bd6aea0d297b331 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/4367cc6e1de5469f9bd6aea0d297b331 2024-11-14T14:56:47,764 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8078e0e84354cb525a824451f0593eef/info of 8078e0e84354cb525a824451f0593eef into 4367cc6e1de5469f9bd6aea0d297b331(size=43.9 K), total size for store is 64.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T14:56:47,765 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8078e0e84354cb525a824451f0593eef: 2024-11-14T14:56:47,765 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef., storeName=8078e0e84354cb525a824451f0593eef/info, priority=13, startTime=1731596207697; duration=0sec 2024-11-14T14:56:47,765 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.6 K, sizeToCheck=16.0 K 2024-11-14T14:56:47,765 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:56:47,765 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/4367cc6e1de5469f9bd6aea0d297b331 because midkey is the same as first or last row 2024-11-14T14:56:47,765 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.6 K, sizeToCheck=16.0 K 2024-11-14T14:56:47,765 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:56:47,765 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/4367cc6e1de5469f9bd6aea0d297b331 because midkey is the same as first or last row 2024-11-14T14:56:47,765 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.6 K, sizeToCheck=16.0 K 2024-11-14T14:56:47,765 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:56:47,765 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/4367cc6e1de5469f9bd6aea0d297b331 because midkey is the same as first or last row 2024-11-14T14:56:47,765 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:56:47,765 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8078e0e84354cb525a824451f0593eef:info 2024-11-14T14:56:48,576 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:48,576 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:49,576 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:49,576 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:49,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8078e0e84354cb525a824451f0593eef 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-11-14T14:56:49,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/0efb09bc468448eaa1ea6a2fd1152d67 is 1080, key is row0053/info:/1731596207699/Put/seqid=0 2024-11-14T14:56:49,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741842_1018 (size=18987) 2024-11-14T14:56:49,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741842_1018 (size=18987) 2024-11-14T14:56:49,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/0efb09bc468448eaa1ea6a2fd1152d67 2024-11-14T14:56:49,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/0efb09bc468448eaa1ea6a2fd1152d67 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/0efb09bc468448eaa1ea6a2fd1152d67 2024-11-14T14:56:49,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/0efb09bc468448eaa1ea6a2fd1152d67, entries=13, sequenceid=82, filesize=18.5 K 2024-11-14T14:56:49,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=16.81 KB/17216 for 8078e0e84354cb525a824451f0593eef in 30ms, sequenceid=82, compaction requested=true 2024-11-14T14:56:49,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8078e0e84354cb525a824451f0593eef: 2024-11-14T14:56:49,754 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=83.1 K, sizeToCheck=16.0 K 2024-11-14T14:56:49,754 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:56:49,754 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/4367cc6e1de5469f9bd6aea0d297b331 because midkey is the same as first or last row 2024-11-14T14:56:49,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8078e0e84354cb525a824451f0593eef:info, priority=-2147483648, current under compaction store size is 1 2024-11-14T14:56:49,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:56:49,755 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T14:56:49,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:49,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8078e0e84354cb525a824451f0593eef 1/1 column families, dataSize=17.86 KB heapSize=19.38 KB 2024-11-14T14:56:49,756 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T14:56:49,756 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1541): 8078e0e84354cb525a824451f0593eef/info is initiating minor compaction (all files) 2024-11-14T14:56:49,756 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8078e0e84354cb525a824451f0593eef/info in TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. 2024-11-14T14:56:49,756 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/4367cc6e1de5469f9bd6aea0d297b331, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/43c916d615d1427b9193d9f08485243a, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/0efb09bc468448eaa1ea6a2fd1152d67] into tmpdir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp, totalSize=83.1 K 2024-11-14T14:56:49,756 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4367cc6e1de5469f9bd6aea0d297b331, keycount=37, bloomtype=ROW, size=43.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731596205604 2024-11-14T14:56:49,757 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43c916d615d1427b9193d9f08485243a, keycount=15, bloomtype=ROW, size=20.6 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731596207671 2024-11-14T14:56:49,757 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0efb09bc468448eaa1ea6a2fd1152d67, keycount=13, bloomtype=ROW, size=18.5 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731596207699 2024-11-14T14:56:49,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/956cdf1b26ef41ae843e91a9fda9b942 is 1080, key is row0066/info:/1731596209726/Put/seqid=0 2024-11-14T14:56:49,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741843_1019 (size=23299) 2024-11-14T14:56:49,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741843_1019 (size=23299) 2024-11-14T14:56:49,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.86 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/956cdf1b26ef41ae843e91a9fda9b942 2024-11-14T14:56:49,771 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8078e0e84354cb525a824451f0593eef#info#compaction#63 average throughput is 33.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T14:56:49,772 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/704ee65e0b84460787443e1bc1ef2c05 is 1080, key is row0001/info:/1731596205604/Put/seqid=0 2024-11-14T14:56:49,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/956cdf1b26ef41ae843e91a9fda9b942 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/956cdf1b26ef41ae843e91a9fda9b942 2024-11-14T14:56:49,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741844_1020 (size=75378) 2024-11-14T14:56:49,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=8078e0e84354cb525a824451f0593eef, server=da1c6afcd1f9,36857,1731596194645 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-14T14:56:49,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/956cdf1b26ef41ae843e91a9fda9b942, entries=17, sequenceid=102, filesize=22.8 K 2024-11-14T14:56:49,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741844_1020 (size=75378) 2024-11-14T14:56:49,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~17.86 KB/18292, heapSize ~19.36 KB/19824, currentSize=12.61 KB/12912 for 8078e0e84354cb525a824451f0593eef in 27ms, sequenceid=102, compaction requested=false 2024-11-14T14:56:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8078e0e84354cb525a824451f0593eef: 2024-11-14T14:56:49,783 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=105.9 K, sizeToCheck=16.0 K 2024-11-14T14:56:49,783 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:56:49,783 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/4367cc6e1de5469f9bd6aea0d297b331 because midkey is the same as first or last row 2024-11-14T14:56:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:38712 deadline: 1731596219779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=8078e0e84354cb525a824451f0593eef, server=da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:49,788 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/704ee65e0b84460787443e1bc1ef2c05 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/704ee65e0b84460787443e1bc1ef2c05 2024-11-14T14:56:49,795 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8078e0e84354cb525a824451f0593eef/info of 8078e0e84354cb525a824451f0593eef into 704ee65e0b84460787443e1bc1ef2c05(size=73.6 K), total size for store is 96.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T14:56:49,795 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8078e0e84354cb525a824451f0593eef: 2024-11-14T14:56:49,795 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef., storeName=8078e0e84354cb525a824451f0593eef/info, priority=13, startTime=1731596209754; duration=0sec 2024-11-14T14:56:49,795 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=96.4 K, sizeToCheck=16.0 K 2024-11-14T14:56:49,795 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:56:49,795 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=96.4 K, sizeToCheck=16.0 K 2024-11-14T14:56:49,795 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:56:49,795 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=96.4 K, sizeToCheck=16.0 K 2024-11-14T14:56:49,795 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-14T14:56:49,796 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:56:49,796 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:56:49,796 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8078e0e84354cb525a824451f0593eef:info 2024-11-14T14:56:49,797 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41179 {}] assignment.AssignmentManager(1363): Split request from da1c6afcd1f9,36857,1731596194645, parent={ENCODED => 8078e0e84354cb525a824451f0593eef, NAME => 'TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062 2024-11-14T14:56:49,803 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41179 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef., hostname=da1c6afcd1f9,36857,1731596194645, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef., hostname=da1c6afcd1f9,36857,1731596194645, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=8078e0e84354cb525a824451f0593eef, server=da1c6afcd1f9,36857,1731596194645 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T14:56:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef., hostname=da1c6afcd1f9,36857,1731596194645, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=8078e0e84354cb525a824451f0593eef, server=da1c6afcd1f9,36857,1731596194645 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T14:56:49,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef., hostname=da1c6afcd1f9,36857,1731596194645, seqNum=2 because the exception is null or not the one we care about 2024-11-14T14:56:49,811 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41179 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8078e0e84354cb525a824451f0593eef, daughterA=fafc242523dc305aa85bb931d0d24ef9, daughterB=c872993953da71467947c204b88318d7 2024-11-14T14:56:49,812 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8078e0e84354cb525a824451f0593eef, daughterA=fafc242523dc305aa85bb931d0d24ef9, daughterB=c872993953da71467947c204b88318d7 2024-11-14T14:56:49,812 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8078e0e84354cb525a824451f0593eef, daughterA=fafc242523dc305aa85bb931d0d24ef9, daughterB=c872993953da71467947c204b88318d7 2024-11-14T14:56:49,812 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8078e0e84354cb525a824451f0593eef, daughterA=fafc242523dc305aa85bb931d0d24ef9, daughterB=c872993953da71467947c204b88318d7 2024-11-14T14:56:49,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8078e0e84354cb525a824451f0593eef, UNASSIGN}] 2024-11-14T14:56:49,821 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8078e0e84354cb525a824451f0593eef, UNASSIGN 2024-11-14T14:56:49,823 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=8078e0e84354cb525a824451f0593eef, regionState=CLOSING, regionLocation=da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:49,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8078e0e84354cb525a824451f0593eef, UNASSIGN because future has completed 2024-11-14T14:56:49,825 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false 2024-11-14T14:56:49,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8078e0e84354cb525a824451f0593eef, server=da1c6afcd1f9,36857,1731596194645}] 2024-11-14T14:56:49,983 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:49,983 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true 2024-11-14T14:56:49,983 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing 8078e0e84354cb525a824451f0593eef, disabling compactions & flushes 2024-11-14T14:56:49,984 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. 2024-11-14T14:56:49,984 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. 2024-11-14T14:56:49,984 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. after waiting 0 ms 2024-11-14T14:56:49,984 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. 2024-11-14T14:56:49,984 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(2902): Flushing 8078e0e84354cb525a824451f0593eef 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-11-14T14:56:49,988 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/71c5a5d0802f4ffc9f81d716fa53fdc6 is 1080, key is row0083/info:/1731596209756/Put/seqid=0 2024-11-14T14:56:49,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741845_1021 (size=17894) 2024-11-14T14:56:49,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741845_1021 (size=17894) 2024-11-14T14:56:49,994 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/71c5a5d0802f4ffc9f81d716fa53fdc6 2024-11-14T14:56:50,000 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/.tmp/info/71c5a5d0802f4ffc9f81d716fa53fdc6 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/71c5a5d0802f4ffc9f81d716fa53fdc6 2024-11-14T14:56:50,006 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/71c5a5d0802f4ffc9f81d716fa53fdc6, entries=12, sequenceid=118, filesize=17.5 K 2024-11-14T14:56:50,007 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=0 B/0 for 8078e0e84354cb525a824451f0593eef in 23ms, sequenceid=118, compaction requested=true 2024-11-14T14:56:50,008 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/1066df47e5b345e58248fc30804d930d, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/6a80e95af09c481092630a399a665bec, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/4367cc6e1de5469f9bd6aea0d297b331, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/177bc9bf45844277af42a42f70b3bf60, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/43c916d615d1427b9193d9f08485243a, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/0efb09bc468448eaa1ea6a2fd1152d67] to archive 2024-11-14T14:56:50,009 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T14:56:50,011 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/1066df47e5b345e58248fc30804d930d to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/1066df47e5b345e58248fc30804d930d 2024-11-14T14:56:50,012 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/6a80e95af09c481092630a399a665bec to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/6a80e95af09c481092630a399a665bec 2024-11-14T14:56:50,013 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/4367cc6e1de5469f9bd6aea0d297b331 to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/4367cc6e1de5469f9bd6aea0d297b331 2024-11-14T14:56:50,014 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/177bc9bf45844277af42a42f70b3bf60 to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/177bc9bf45844277af42a42f70b3bf60 2024-11-14T14:56:50,016 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/43c916d615d1427b9193d9f08485243a to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/43c916d615d1427b9193d9f08485243a 2024-11-14T14:56:50,017 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/0efb09bc468448eaa1ea6a2fd1152d67 to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/0efb09bc468448eaa1ea6a2fd1152d67 2024-11-14T14:56:50,023 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/recovered.edits/121.seqid, newMaxSeqId=121, maxSeqId=1 2024-11-14T14:56:50,024 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. 2024-11-14T14:56:50,025 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for 8078e0e84354cb525a824451f0593eef: Waiting for close lock at 1731596209983Running coprocessor pre-close hooks at 1731596209983Disabling compacts and flushes for region at 1731596209983Disabling writes for close at 1731596209984 (+1 ms)Obtaining lock to block concurrent updates at 1731596209984Preparing flush snapshotting stores in 8078e0e84354cb525a824451f0593eef at 1731596209984Finished memstore snapshotting TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef., syncing WAL and waiting on mvcc, flushsize=dataSize=12912, getHeapSize=14064, getOffHeapSize=0, getCellsCount=12 at 1731596209984Flushing stores of TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. at 1731596209985 (+1 ms)Flushing 8078e0e84354cb525a824451f0593eef/info: creating writer at 1731596209985Flushing 8078e0e84354cb525a824451f0593eef/info: appending metadata at 1731596209988 (+3 ms)Flushing 8078e0e84354cb525a824451f0593eef/info: closing flushed file at 1731596209988Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2682a218: reopening flushed file at 1731596209999 (+11 ms)Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=0 B/0 for 8078e0e84354cb525a824451f0593eef in 23ms, sequenceid=118, compaction requested=true at 1731596210007 (+8 ms)Writing region close event to WAL at 1731596210019 (+12 ms)Running coprocessor post-close hooks at 1731596210024 (+5 ms)Closed at 1731596210024 2024-11-14T14:56:50,027 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:50,028 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=8078e0e84354cb525a824451f0593eef, regionState=CLOSED 2024-11-14T14:56:50,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8078e0e84354cb525a824451f0593eef, server=da1c6afcd1f9,36857,1731596194645 because future has completed 2024-11-14T14:56:50,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-14T14:56:50,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure 8078e0e84354cb525a824451f0593eef, server=da1c6afcd1f9,36857,1731596194645 in 205 msec 2024-11-14T14:56:50,036 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T14:56:50,036 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=8078e0e84354cb525a824451f0593eef, UNASSIGN in 214 msec 2024-11-14T14:56:50,046 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:50,050 INFO [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 3 storefiles, region=8078e0e84354cb525a824451f0593eef, threads=3 2024-11-14T14:56:50,052 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/71c5a5d0802f4ffc9f81d716fa53fdc6 for region: 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:50,052 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/704ee65e0b84460787443e1bc1ef2c05 for region: 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:50,052 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/956cdf1b26ef41ae843e91a9fda9b942 for region: 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:50,063 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/71c5a5d0802f4ffc9f81d716fa53fdc6, top=true 2024-11-14T14:56:50,064 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/956cdf1b26ef41ae843e91a9fda9b942, top=true 2024-11-14T14:56:50,068 INFO [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/TestLogRolling-testLogRolling=8078e0e84354cb525a824451f0593eef-71c5a5d0802f4ffc9f81d716fa53fdc6 for child: c872993953da71467947c204b88318d7, parent: 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:50,069 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/71c5a5d0802f4ffc9f81d716fa53fdc6 for region: 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:50,073 INFO [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/TestLogRolling-testLogRolling=8078e0e84354cb525a824451f0593eef-956cdf1b26ef41ae843e91a9fda9b942 for child: c872993953da71467947c204b88318d7, parent: 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:50,073 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/956cdf1b26ef41ae843e91a9fda9b942 for region: 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:50,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741846_1022 (size=27) 2024-11-14T14:56:50,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741846_1022 (size=27) 2024-11-14T14:56:50,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741847_1023 (size=27) 2024-11-14T14:56:50,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741847_1023 (size=27) 2024-11-14T14:56:50,088 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/704ee65e0b84460787443e1bc1ef2c05 for region: 8078e0e84354cb525a824451f0593eef 2024-11-14T14:56:50,090 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 8078e0e84354cb525a824451f0593eef Daughter A: [hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/fafc242523dc305aa85bb931d0d24ef9/info/704ee65e0b84460787443e1bc1ef2c05.8078e0e84354cb525a824451f0593eef] storefiles, Daughter B: [hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/704ee65e0b84460787443e1bc1ef2c05.8078e0e84354cb525a824451f0593eef, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/TestLogRolling-testLogRolling=8078e0e84354cb525a824451f0593eef-71c5a5d0802f4ffc9f81d716fa53fdc6, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/TestLogRolling-testLogRolling=8078e0e84354cb525a824451f0593eef-956cdf1b26ef41ae843e91a9fda9b942] storefiles. 2024-11-14T14:56:50,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741848_1024 (size=71) 2024-11-14T14:56:50,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741848_1024 (size=71) 2024-11-14T14:56:50,100 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:50,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741849_1025 (size=71) 2024-11-14T14:56:50,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741849_1025 (size=71) 2024-11-14T14:56:50,114 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:50,124 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/fafc242523dc305aa85bb931d0d24ef9/recovered.edits/121.seqid, newMaxSeqId=121, maxSeqId=-1 2024-11-14T14:56:50,126 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/recovered.edits/121.seqid, newMaxSeqId=121, maxSeqId=-1 2024-11-14T14:56:50,129 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731596210128"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1731596210128"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1731596210128"}]},"ts":"1731596210128"} 2024-11-14T14:56:50,129 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731596210128"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731596210128"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731596210128"}]},"ts":"1731596210128"} 2024-11-14T14:56:50,129 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731596210128"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731596210128"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731596210128"}]},"ts":"1731596210128"} 2024-11-14T14:56:50,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fafc242523dc305aa85bb931d0d24ef9, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c872993953da71467947c204b88318d7, ASSIGN}] 2024-11-14T14:56:50,152 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c872993953da71467947c204b88318d7, ASSIGN 2024-11-14T14:56:50,152 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fafc242523dc305aa85bb931d0d24ef9, ASSIGN 2024-11-14T14:56:50,153 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c872993953da71467947c204b88318d7, ASSIGN; state=SPLITTING_NEW, location=da1c6afcd1f9,36857,1731596194645; forceNewPlan=false, retain=false 2024-11-14T14:56:50,153 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fafc242523dc305aa85bb931d0d24ef9, ASSIGN; state=SPLITTING_NEW, location=da1c6afcd1f9,36857,1731596194645; forceNewPlan=false, retain=false 2024-11-14T14:56:50,304 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=c872993953da71467947c204b88318d7, regionState=OPENING, regionLocation=da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:50,304 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fafc242523dc305aa85bb931d0d24ef9, regionState=OPENING, regionLocation=da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:50,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c872993953da71467947c204b88318d7, ASSIGN because future has completed 2024-11-14T14:56:50,307 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure c872993953da71467947c204b88318d7, server=da1c6afcd1f9,36857,1731596194645}] 2024-11-14T14:56:50,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fafc242523dc305aa85bb931d0d24ef9, ASSIGN because future has completed 2024-11-14T14:56:50,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure fafc242523dc305aa85bb931d0d24ef9, server=da1c6afcd1f9,36857,1731596194645}] 2024-11-14T14:56:50,463 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7. 2024-11-14T14:56:50,463 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => c872993953da71467947c204b88318d7, NAME => 'TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.', STARTKEY => 'row0062', ENDKEY => ''} 2024-11-14T14:56:50,464 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling c872993953da71467947c204b88318d7 2024-11-14T14:56:50,464 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:56:50,464 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for c872993953da71467947c204b88318d7 2024-11-14T14:56:50,464 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for c872993953da71467947c204b88318d7 2024-11-14T14:56:50,465 INFO [StoreOpener-c872993953da71467947c204b88318d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region c872993953da71467947c204b88318d7 2024-11-14T14:56:50,466 INFO [StoreOpener-c872993953da71467947c204b88318d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c872993953da71467947c204b88318d7 columnFamilyName info 2024-11-14T14:56:50,466 DEBUG [StoreOpener-c872993953da71467947c204b88318d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:50,477 DEBUG [StoreOpener-c872993953da71467947c204b88318d7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/704ee65e0b84460787443e1bc1ef2c05.8078e0e84354cb525a824451f0593eef->hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/704ee65e0b84460787443e1bc1ef2c05-top 2024-11-14T14:56:50,481 DEBUG [StoreOpener-c872993953da71467947c204b88318d7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/TestLogRolling-testLogRolling=8078e0e84354cb525a824451f0593eef-71c5a5d0802f4ffc9f81d716fa53fdc6 2024-11-14T14:56:50,486 DEBUG [StoreOpener-c872993953da71467947c204b88318d7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/TestLogRolling-testLogRolling=8078e0e84354cb525a824451f0593eef-956cdf1b26ef41ae843e91a9fda9b942 2024-11-14T14:56:50,486 INFO [StoreOpener-c872993953da71467947c204b88318d7-1 {}] regionserver.HStore(327): Store=c872993953da71467947c204b88318d7/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:56:50,486 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for c872993953da71467947c204b88318d7 2024-11-14T14:56:50,487 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7 2024-11-14T14:56:50,488 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7 2024-11-14T14:56:50,488 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for c872993953da71467947c204b88318d7 2024-11-14T14:56:50,488 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for c872993953da71467947c204b88318d7 2024-11-14T14:56:50,491 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for c872993953da71467947c204b88318d7 2024-11-14T14:56:50,491 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened c872993953da71467947c204b88318d7; next sequenceid=122; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=706104, jitterRate=-0.10214294493198395}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:56:50,491 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c872993953da71467947c204b88318d7 2024-11-14T14:56:50,492 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for c872993953da71467947c204b88318d7: Running coprocessor pre-open hook at 1731596210464Writing region info on filesystem at 1731596210464Initializing all the Stores at 1731596210465 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596210465Cleaning up temporary data from old regions at 1731596210488 (+23 ms)Running coprocessor post-open hooks at 1731596210492 (+4 ms)Region opened successfully at 1731596210492 2024-11-14T14:56:50,493 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7., pid=12, masterSystemTime=1731596210459 2024-11-14T14:56:50,494 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store c872993953da71467947c204b88318d7:info, priority=-2147483648, current under compaction store size is 1 2024-11-14T14:56:50,494 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:56:50,494 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T14:56:50,495 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7. 2024-11-14T14:56:50,495 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1541): c872993953da71467947c204b88318d7/info is initiating minor compaction (all files) 2024-11-14T14:56:50,495 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c872993953da71467947c204b88318d7/info in TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7. 2024-11-14T14:56:50,495 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/704ee65e0b84460787443e1bc1ef2c05.8078e0e84354cb525a824451f0593eef->hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/704ee65e0b84460787443e1bc1ef2c05-top, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/TestLogRolling-testLogRolling=8078e0e84354cb525a824451f0593eef-956cdf1b26ef41ae843e91a9fda9b942, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/TestLogRolling-testLogRolling=8078e0e84354cb525a824451f0593eef-71c5a5d0802f4ffc9f81d716fa53fdc6] into tmpdir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp, totalSize=113.8 K 2024-11-14T14:56:50,496 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7. 2024-11-14T14:56:50,496 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7. 2024-11-14T14:56:50,496 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 704ee65e0b84460787443e1bc1ef2c05.8078e0e84354cb525a824451f0593eef, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731596205604 2024-11-14T14:56:50,496 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9. 2024-11-14T14:56:50,496 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => fafc242523dc305aa85bb931d0d24ef9, NAME => 'TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9.', STARTKEY => '', ENDKEY => 'row0062'} 2024-11-14T14:56:50,496 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=8078e0e84354cb525a824451f0593eef-956cdf1b26ef41ae843e91a9fda9b942, keycount=17, bloomtype=ROW, size=22.8 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731596209726 2024-11-14T14:56:50,496 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling fafc242523dc305aa85bb931d0d24ef9 2024-11-14T14:56:50,497 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:56:50,497 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=c872993953da71467947c204b88318d7, regionState=OPEN, openSeqNum=122, regionLocation=da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:50,497 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for fafc242523dc305aa85bb931d0d24ef9 2024-11-14T14:56:50,497 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for fafc242523dc305aa85bb931d0d24ef9 2024-11-14T14:56:50,497 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=8078e0e84354cb525a824451f0593eef-71c5a5d0802f4ffc9f81d716fa53fdc6, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731596209756 2024-11-14T14:56:50,498 INFO [StoreOpener-fafc242523dc305aa85bb931d0d24ef9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region fafc242523dc305aa85bb931d0d24ef9 2024-11-14T14:56:50,499 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on 1588230740 2024-11-14T14:56:50,499 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-14T14:56:50,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.15 KB heapSize=9 KB 2024-11-14T14:56:50,499 INFO [StoreOpener-fafc242523dc305aa85bb931d0d24ef9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fafc242523dc305aa85bb931d0d24ef9 columnFamilyName info 2024-11-14T14:56:50,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure c872993953da71467947c204b88318d7, server=da1c6afcd1f9,36857,1731596194645 because future has completed 2024-11-14T14:56:50,499 DEBUG [StoreOpener-fafc242523dc305aa85bb931d0d24ef9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:56:50,504 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-14T14:56:50,504 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure c872993953da71467947c204b88318d7, server=da1c6afcd1f9,36857,1731596194645 in 194 msec 2024-11-14T14:56:50,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=c872993953da71467947c204b88318d7, ASSIGN in 353 msec 2024-11-14T14:56:50,511 DEBUG [StoreOpener-fafc242523dc305aa85bb931d0d24ef9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/fafc242523dc305aa85bb931d0d24ef9/info/704ee65e0b84460787443e1bc1ef2c05.8078e0e84354cb525a824451f0593eef->hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/704ee65e0b84460787443e1bc1ef2c05-bottom 2024-11-14T14:56:50,511 INFO [StoreOpener-fafc242523dc305aa85bb931d0d24ef9-1 {}] regionserver.HStore(327): Store=fafc242523dc305aa85bb931d0d24ef9/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:56:50,511 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for fafc242523dc305aa85bb931d0d24ef9 2024-11-14T14:56:50,512 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/fafc242523dc305aa85bb931d0d24ef9 2024-11-14T14:56:50,513 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/fafc242523dc305aa85bb931d0d24ef9 2024-11-14T14:56:50,514 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for fafc242523dc305aa85bb931d0d24ef9 2024-11-14T14:56:50,514 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for fafc242523dc305aa85bb931d0d24ef9 2024-11-14T14:56:50,516 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for fafc242523dc305aa85bb931d0d24ef9 2024-11-14T14:56:50,517 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened fafc242523dc305aa85bb931d0d24ef9; next sequenceid=122; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=854150, jitterRate=0.08610890805721283}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T14:56:50,517 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fafc242523dc305aa85bb931d0d24ef9 2024-11-14T14:56:50,517 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for fafc242523dc305aa85bb931d0d24ef9: Running coprocessor pre-open hook at 1731596210497Writing region info on filesystem at 1731596210497Initializing all the Stores at 1731596210498 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596210498Cleaning up temporary data from old regions at 1731596210514 (+16 ms)Running coprocessor post-open hooks at 1731596210517 (+3 ms)Region opened successfully at 1731596210517 2024-11-14T14:56:50,518 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9., pid=13, masterSystemTime=1731596210459 2024-11-14T14:56:50,518 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store fafc242523dc305aa85bb931d0d24ef9:info, priority=-2147483648, current under compaction store size is 2 2024-11-14T14:56:50,518 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:56:50,518 DEBUG [RS:0;da1c6afcd1f9:36857-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-14T14:56:50,519 INFO [RS:0;da1c6afcd1f9:36857-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9. 2024-11-14T14:56:50,519 DEBUG [RS:0;da1c6afcd1f9:36857-longCompactions-0 {}] regionserver.HStore(1541): fafc242523dc305aa85bb931d0d24ef9/info is initiating minor compaction (all files) 2024-11-14T14:56:50,519 INFO [RS:0;da1c6afcd1f9:36857-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fafc242523dc305aa85bb931d0d24ef9/info in TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9. 2024-11-14T14:56:50,520 INFO [RS:0;da1c6afcd1f9:36857-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/fafc242523dc305aa85bb931d0d24ef9/info/704ee65e0b84460787443e1bc1ef2c05.8078e0e84354cb525a824451f0593eef->hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/704ee65e0b84460787443e1bc1ef2c05-bottom] into tmpdir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/fafc242523dc305aa85bb931d0d24ef9/.tmp, totalSize=73.6 K 2024-11-14T14:56:50,520 DEBUG [RS:0;da1c6afcd1f9:36857-longCompactions-0 {}] compactions.Compactor(225): Compacting 704ee65e0b84460787443e1bc1ef2c05.8078e0e84354cb525a824451f0593eef, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731596205604 2024-11-14T14:56:50,521 DEBUG [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9. 2024-11-14T14:56:50,521 INFO [RS_OPEN_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9. 2024-11-14T14:56:50,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/.tmp/info/cdb1c1313aa842178c9316503ac4ab7d is 193, key is TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7./info:regioninfo/1731596210496/Put/seqid=0 2024-11-14T14:56:50,521 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fafc242523dc305aa85bb931d0d24ef9, regionState=OPEN, openSeqNum=122, regionLocation=da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:50,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure fafc242523dc305aa85bb931d0d24ef9, server=da1c6afcd1f9,36857,1731596194645 because future has completed 2024-11-14T14:56:50,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=10 2024-11-14T14:56:50,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure fafc242523dc305aa85bb931d0d24ef9, server=da1c6afcd1f9,36857,1731596194645 in 217 msec 2024-11-14T14:56:50,530 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c872993953da71467947c204b88318d7#info#compaction#66 average throughput is 11.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T14:56:50,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-14T14:56:50,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=fafc242523dc305aa85bb931d0d24ef9, ASSIGN in 376 msec 2024-11-14T14:56:50,531 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/62e4107d4f1c4357aa8e725e15efeb2d is 1080, key is row0062/info:/1731596207716/Put/seqid=0 2024-11-14T14:56:50,533 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=8078e0e84354cb525a824451f0593eef, daughterA=fafc242523dc305aa85bb931d0d24ef9, daughterB=c872993953da71467947c204b88318d7 in 728 msec 2024-11-14T14:56:50,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741850_1026 (size=9882) 2024-11-14T14:56:50,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741850_1026 (size=9882) 2024-11-14T14:56:50,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.95 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/.tmp/info/cdb1c1313aa842178c9316503ac4ab7d 2024-11-14T14:56:50,545 INFO [RS:0;da1c6afcd1f9:36857-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fafc242523dc305aa85bb931d0d24ef9#info#compaction#67 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T14:56:50,546 DEBUG [RS:0;da1c6afcd1f9:36857-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/fafc242523dc305aa85bb931d0d24ef9/.tmp/info/00df06cc89bd4f52bdb9bb848e4369ff is 1080, key is row0001/info:/1731596205604/Put/seqid=0 2024-11-14T14:56:50,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741851_1027 (size=40830) 2024-11-14T14:56:50,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741851_1027 (size=40830) 2024-11-14T14:56:50,554 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/62e4107d4f1c4357aa8e725e15efeb2d as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/62e4107d4f1c4357aa8e725e15efeb2d 2024-11-14T14:56:50,560 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c872993953da71467947c204b88318d7/info of c872993953da71467947c204b88318d7 into 62e4107d4f1c4357aa8e725e15efeb2d(size=39.9 K), total size for store is 39.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T14:56:50,560 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:56:50,560 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7., storeName=c872993953da71467947c204b88318d7/info, priority=13, startTime=1731596210493; duration=0sec 2024-11-14T14:56:50,560 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:56:50,560 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c872993953da71467947c204b88318d7:info 2024-11-14T14:56:50,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741852_1028 (size=70862) 2024-11-14T14:56:50,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741852_1028 (size=70862) 2024-11-14T14:56:50,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/.tmp/ns/7568f970b2b141a1a3c83bba8b77b1f1 is 43, key is default/ns:d/1731596195457/Put/seqid=0 2024-11-14T14:56:50,571 DEBUG [RS:0;da1c6afcd1f9:36857-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/fafc242523dc305aa85bb931d0d24ef9/.tmp/info/00df06cc89bd4f52bdb9bb848e4369ff as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/fafc242523dc305aa85bb931d0d24ef9/info/00df06cc89bd4f52bdb9bb848e4369ff 2024-11-14T14:56:50,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741853_1029 (size=5153) 2024-11-14T14:56:50,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741853_1029 (size=5153) 2024-11-14T14:56:50,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/.tmp/ns/7568f970b2b141a1a3c83bba8b77b1f1 2024-11-14T14:56:50,577 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:50,577 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:50,578 INFO [RS:0;da1c6afcd1f9:36857-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in fafc242523dc305aa85bb931d0d24ef9/info of fafc242523dc305aa85bb931d0d24ef9 into 00df06cc89bd4f52bdb9bb848e4369ff(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T14:56:50,578 DEBUG [RS:0;da1c6afcd1f9:36857-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fafc242523dc305aa85bb931d0d24ef9: 2024-11-14T14:56:50,578 INFO [RS:0;da1c6afcd1f9:36857-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9., storeName=fafc242523dc305aa85bb931d0d24ef9/info, priority=15, startTime=1731596210518; duration=0sec 2024-11-14T14:56:50,578 DEBUG [RS:0;da1c6afcd1f9:36857-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:56:50,578 DEBUG [RS:0;da1c6afcd1f9:36857-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fafc242523dc305aa85bb931d0d24ef9:info 2024-11-14T14:56:50,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/.tmp/table/52c38e58b6254c288d551201fd12b555 is 65, key is TestLogRolling-testLogRolling/table:state/1731596195943/Put/seqid=0 2024-11-14T14:56:50,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741854_1030 (size=5340) 2024-11-14T14:56:50,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741854_1030 (size=5340) 2024-11-14T14:56:50,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/.tmp/table/52c38e58b6254c288d551201fd12b555 2024-11-14T14:56:50,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/.tmp/info/cdb1c1313aa842178c9316503ac4ab7d as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/info/cdb1c1313aa842178c9316503ac4ab7d 2024-11-14T14:56:50,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/info/cdb1c1313aa842178c9316503ac4ab7d, entries=30, sequenceid=17, filesize=9.7 K 2024-11-14T14:56:50,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/.tmp/ns/7568f970b2b141a1a3c83bba8b77b1f1 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/ns/7568f970b2b141a1a3c83bba8b77b1f1 2024-11-14T14:56:50,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/ns/7568f970b2b141a1a3c83bba8b77b1f1, entries=2, sequenceid=17, filesize=5.0 K 2024-11-14T14:56:50,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/.tmp/table/52c38e58b6254c288d551201fd12b555 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/table/52c38e58b6254c288d551201fd12b555 2024-11-14T14:56:50,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/table/52c38e58b6254c288d551201fd12b555, entries=2, sequenceid=17, filesize=5.2 K 2024-11-14T14:56:50,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.15 KB/5269, heapSize ~8.70 KB/8912, currentSize=670 B/670 for 1588230740 in 122ms, sequenceid=17, compaction requested=false 2024-11-14T14:56:50,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-14T14:56:51,578 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:51,578 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:52,578 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:52,578 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:53,579 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:53,579 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:54,579 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:54,579 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:55,527 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-14T14:56:55,527 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:55,528 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:55,528 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:55,528 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:55,528 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:55,528 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:55,529 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:55,529 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:55,547 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:55,548 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:55,548 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:55,548 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:55,549 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:55,549 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:55,552 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:55,553 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:55,553 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:55,555 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-14T14:56:55,580 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:55,580 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:56,581 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:56,581 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:57,581 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:57,581 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:58,582 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:58,582 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:59,583 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:59,583 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:56:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:38712 deadline: 1731596229917, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. is not online on da1c6afcd1f9,36857,1731596194645 2024-11-14T14:56:59,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef., hostname=da1c6afcd1f9,36857,1731596194645, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef., hostname=da1c6afcd1f9,36857,1731596194645, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. is not online on da1c6afcd1f9,36857,1731596194645 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T14:56:59,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef., hostname=da1c6afcd1f9,36857,1731596194645, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef. is not online on da1c6afcd1f9,36857,1731596194645 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T14:56:59,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1731596195575.8078e0e84354cb525a824451f0593eef., hostname=da1c6afcd1f9,36857,1731596194645, seqNum=2 from cache 2024-11-14T14:57:00,583 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:00,583 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:01,584 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:01,584 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:02,585 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:02,585 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:03,585 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:03,585 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:04,575 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T14:57:04,586 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:04,586 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:05,586 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:05,586 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:06,587 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:06,587 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:07,588 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:07,588 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:08,588 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:08,588 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:09,589 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:09,589 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:10,590 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:10,590 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:11,590 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:11,590 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:12,591 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:12,591 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:13,592 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:13,592 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:14,592 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:14,592 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:15,515 INFO [master/da1c6afcd1f9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-14T14:57:15,515 INFO [master/da1c6afcd1f9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-14T14:57:15,593 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:15,593 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:16,594 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:16,594 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:17,594 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:17,594 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:18,595 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:18,595 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:19,595 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:19,595 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:20,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0095', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7., hostname=da1c6afcd1f9,36857,1731596194645, seqNum=122] 2024-11-14T14:57:20,424 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20375 2024-11-14T14:57:20,596 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:20,596 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:21,597 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:21,597 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on c872993953da71467947c204b88318d7 2024-11-14T14:57:22,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c872993953da71467947c204b88318d7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-14T14:57:22,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/0c3add6a228740ecb2c25c7fbce97d47 is 1080, key is row0095/info:/1731596240054/Put/seqid=0 2024-11-14T14:57:22,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741855_1031 (size=12513) 2024-11-14T14:57:22,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741855_1031 (size=12513) 2024-11-14T14:57:22,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/0c3add6a228740ecb2c25c7fbce97d47 2024-11-14T14:57:22,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/0c3add6a228740ecb2c25c7fbce97d47 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/0c3add6a228740ecb2c25c7fbce97d47 2024-11-14T14:57:22,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/0c3add6a228740ecb2c25c7fbce97d47, entries=7, sequenceid=132, filesize=12.2 K 2024-11-14T14:57:22,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=21.02 KB/21520 for c872993953da71467947c204b88318d7 in 37ms, sequenceid=132, compaction requested=false 2024-11-14T14:57:22,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on c872993953da71467947c204b88318d7 2024-11-14T14:57:22,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c872993953da71467947c204b88318d7 1/1 column families, dataSize=22.07 KB heapSize=23.88 KB 2024-11-14T14:57:22,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/bc60517d583a4c1c9ab6db0a1a890c4a is 1080, key is row0102/info:/1731596242068/Put/seqid=0 2024-11-14T14:57:22,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741856_1032 (size=27628) 2024-11-14T14:57:22,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741856_1032 (size=27628) 2024-11-14T14:57:22,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.07 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/bc60517d583a4c1c9ab6db0a1a890c4a 2024-11-14T14:57:22,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/bc60517d583a4c1c9ab6db0a1a890c4a as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/bc60517d583a4c1c9ab6db0a1a890c4a 2024-11-14T14:57:22,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/bc60517d583a4c1c9ab6db0a1a890c4a, entries=21, sequenceid=156, filesize=27.0 K 2024-11-14T14:57:22,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~22.07 KB/22596, heapSize ~23.86 KB/24432, currentSize=6.30 KB/6456 for c872993953da71467947c204b88318d7 in 24ms, sequenceid=156, compaction requested=true 2024-11-14T14:57:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c872993953da71467947c204b88318d7:info, priority=-2147483648, current under compaction store size is 1 2024-11-14T14:57:22,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:57:22,129 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T14:57:22,130 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 80971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T14:57:22,130 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1541): c872993953da71467947c204b88318d7/info is initiating minor compaction (all files) 2024-11-14T14:57:22,130 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c872993953da71467947c204b88318d7/info in TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7. 2024-11-14T14:57:22,130 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/62e4107d4f1c4357aa8e725e15efeb2d, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/0c3add6a228740ecb2c25c7fbce97d47, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/bc60517d583a4c1c9ab6db0a1a890c4a] into tmpdir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp, totalSize=79.1 K 2024-11-14T14:57:22,131 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62e4107d4f1c4357aa8e725e15efeb2d, keycount=33, bloomtype=ROW, size=39.9 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731596207716 2024-11-14T14:57:22,131 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c3add6a228740ecb2c25c7fbce97d47, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731596240054 2024-11-14T14:57:22,132 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc60517d583a4c1c9ab6db0a1a890c4a, keycount=21, bloomtype=ROW, size=27.0 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731596242068 2024-11-14T14:57:22,142 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c872993953da71467947c204b88318d7#info#compaction#72 average throughput is 62.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T14:57:22,143 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/75afe4e1bda74ff8950fbb14702a5ad9 is 1080, key is row0062/info:/1731596207716/Put/seqid=0 2024-11-14T14:57:22,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741857_1033 (size=71185) 2024-11-14T14:57:22,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741857_1033 (size=71185) 2024-11-14T14:57:22,155 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/75afe4e1bda74ff8950fbb14702a5ad9 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/75afe4e1bda74ff8950fbb14702a5ad9 2024-11-14T14:57:22,162 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c872993953da71467947c204b88318d7/info of c872993953da71467947c204b88318d7 into 75afe4e1bda74ff8950fbb14702a5ad9(size=69.5 K), total size for store is 69.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T14:57:22,162 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:22,162 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7., storeName=c872993953da71467947c204b88318d7/info, priority=13, startTime=1731596242129; duration=0sec 2024-11-14T14:57:22,163 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:57:22,163 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c872993953da71467947c204b88318d7:info 2024-11-14T14:57:22,597 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:22,597 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:23,598 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:23,598 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:24,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on c872993953da71467947c204b88318d7 2024-11-14T14:57:24,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c872993953da71467947c204b88318d7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-14T14:57:24,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/12bea0b58b734e21bfb966763354eca2 is 1080, key is row0123/info:/1731596242106/Put/seqid=0 2024-11-14T14:57:24,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741858_1034 (size=12516) 2024-11-14T14:57:24,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741858_1034 (size=12516) 2024-11-14T14:57:24,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/12bea0b58b734e21bfb966763354eca2 2024-11-14T14:57:24,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/12bea0b58b734e21bfb966763354eca2 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/12bea0b58b734e21bfb966763354eca2 2024-11-14T14:57:24,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/12bea0b58b734e21bfb966763354eca2, entries=7, sequenceid=167, filesize=12.2 K 2024-11-14T14:57:24,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=15.76 KB/16140 for c872993953da71467947c204b88318d7 in 42ms, sequenceid=167, compaction requested=false 2024-11-14T14:57:24,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on c872993953da71467947c204b88318d7 2024-11-14T14:57:24,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c872993953da71467947c204b88318d7 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB 2024-11-14T14:57:24,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/bfc0970175f9453fab0f0cbda3e3346f is 1080, key is row0130/info:/1731596244120/Put/seqid=0 2024-11-14T14:57:24,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741859_1035 (size=22238) 2024-11-14T14:57:24,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741859_1035 (size=22238) 2024-11-14T14:57:24,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/bfc0970175f9453fab0f0cbda3e3346f 2024-11-14T14:57:24,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/bfc0970175f9453fab0f0cbda3e3346f as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/bfc0970175f9453fab0f0cbda3e3346f 2024-11-14T14:57:24,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/bfc0970175f9453fab0f0cbda3e3346f, entries=16, sequenceid=186, filesize=21.7 K 2024-11-14T14:57:24,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=13.66 KB/13988 for c872993953da71467947c204b88318d7 in 24ms, sequenceid=186, compaction requested=true 2024-11-14T14:57:24,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:24,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c872993953da71467947c204b88318d7:info, priority=-2147483648, current under compaction store size is 1 2024-11-14T14:57:24,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:57:24,186 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T14:57:24,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on c872993953da71467947c204b88318d7 2024-11-14T14:57:24,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c872993953da71467947c204b88318d7 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-14T14:57:24,187 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 105939 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T14:57:24,187 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1541): c872993953da71467947c204b88318d7/info is initiating minor compaction (all files) 2024-11-14T14:57:24,188 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c872993953da71467947c204b88318d7/info in TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7. 2024-11-14T14:57:24,188 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/75afe4e1bda74ff8950fbb14702a5ad9, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/12bea0b58b734e21bfb966763354eca2, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/bfc0970175f9453fab0f0cbda3e3346f] into tmpdir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp, totalSize=103.5 K 2024-11-14T14:57:24,188 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75afe4e1bda74ff8950fbb14702a5ad9, keycount=61, bloomtype=ROW, size=69.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731596207716 2024-11-14T14:57:24,189 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12bea0b58b734e21bfb966763354eca2, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731596242106 2024-11-14T14:57:24,189 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfc0970175f9453fab0f0cbda3e3346f, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731596244120 2024-11-14T14:57:24,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/bedcd5ba35d94366a94fd0c8e982108a is 1080, key is row0146/info:/1731596244162/Put/seqid=0 2024-11-14T14:57:24,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741860_1036 (size=20078) 2024-11-14T14:57:24,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741860_1036 (size=20078) 2024-11-14T14:57:24,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/bedcd5ba35d94366a94fd0c8e982108a 2024-11-14T14:57:24,219 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c872993953da71467947c204b88318d7#info#compaction#76 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T14:57:24,220 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/6e41f61029954d8296be7606807aeba1 is 1080, key is row0062/info:/1731596207716/Put/seqid=0 2024-11-14T14:57:24,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741861_1037 (size=96154) 2024-11-14T14:57:24,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/bedcd5ba35d94366a94fd0c8e982108a as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/bedcd5ba35d94366a94fd0c8e982108a 2024-11-14T14:57:24,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741861_1037 (size=96154) 2024-11-14T14:57:24,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/bedcd5ba35d94366a94fd0c8e982108a, entries=14, sequenceid=203, filesize=19.6 K 2024-11-14T14:57:24,231 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/6e41f61029954d8296be7606807aeba1 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/6e41f61029954d8296be7606807aeba1 2024-11-14T14:57:24,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=1.05 KB/1076 for c872993953da71467947c204b88318d7 in 45ms, sequenceid=203, compaction requested=false 2024-11-14T14:57:24,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:24,237 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c872993953da71467947c204b88318d7/info of c872993953da71467947c204b88318d7 into 6e41f61029954d8296be7606807aeba1(size=93.9 K), total size for store is 113.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T14:57:24,237 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:24,237 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7., storeName=c872993953da71467947c204b88318d7/info, priority=13, startTime=1731596244186; duration=0sec 2024-11-14T14:57:24,237 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:57:24,237 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c872993953da71467947c204b88318d7:info 2024-11-14T14:57:24,598 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:24,598 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:25,599 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:25,599 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on c872993953da71467947c204b88318d7 2024-11-14T14:57:26,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c872993953da71467947c204b88318d7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-14T14:57:26,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/3bd4ca95e9114c678c00f8d1bf37c8e0 is 1080, key is row0160/info:/1731596244188/Put/seqid=0 2024-11-14T14:57:26,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741862_1038 (size=12516) 2024-11-14T14:57:26,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741862_1038 (size=12516) 2024-11-14T14:57:26,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/3bd4ca95e9114c678c00f8d1bf37c8e0 2024-11-14T14:57:26,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/3bd4ca95e9114c678c00f8d1bf37c8e0 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/3bd4ca95e9114c678c00f8d1bf37c8e0 2024-11-14T14:57:26,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/3bd4ca95e9114c678c00f8d1bf37c8e0, entries=7, sequenceid=214, filesize=12.2 K 2024-11-14T14:57:26,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=17.86 KB/18292 for c872993953da71467947c204b88318d7 in 29ms, sequenceid=214, compaction requested=true 2024-11-14T14:57:26,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:26,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c872993953da71467947c204b88318d7:info, priority=-2147483648, current under compaction store size is 1 2024-11-14T14:57:26,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:57:26,230 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T14:57:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on c872993953da71467947c204b88318d7 2024-11-14T14:57:26,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c872993953da71467947c204b88318d7 1/1 column families, dataSize=18.91 KB heapSize=20.50 KB 2024-11-14T14:57:26,231 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128748 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T14:57:26,231 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1541): c872993953da71467947c204b88318d7/info is initiating minor compaction (all files) 2024-11-14T14:57:26,231 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c872993953da71467947c204b88318d7/info in TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7. 2024-11-14T14:57:26,231 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/6e41f61029954d8296be7606807aeba1, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/bedcd5ba35d94366a94fd0c8e982108a, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/3bd4ca95e9114c678c00f8d1bf37c8e0] into tmpdir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp, totalSize=125.7 K 2024-11-14T14:57:26,232 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e41f61029954d8296be7606807aeba1, keycount=84, bloomtype=ROW, size=93.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731596207716 2024-11-14T14:57:26,233 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting bedcd5ba35d94366a94fd0c8e982108a, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731596244162 2024-11-14T14:57:26,233 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bd4ca95e9114c678c00f8d1bf37c8e0, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731596244188 2024-11-14T14:57:26,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/cdc9149df98a4b9d9817a23f9a9cd95c is 1080, key is row0167/info:/1731596246202/Put/seqid=0 2024-11-14T14:57:26,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741863_1039 (size=24394) 2024-11-14T14:57:26,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741863_1039 (size=24394) 2024-11-14T14:57:26,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.91 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/cdc9149df98a4b9d9817a23f9a9cd95c 2024-11-14T14:57:26,247 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c872993953da71467947c204b88318d7#info#compaction#79 average throughput is 53.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T14:57:26,247 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/0ff1d1c5eae94dda8638d77c2437544a is 1080, key is row0062/info:/1731596207716/Put/seqid=0 2024-11-14T14:57:26,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/cdc9149df98a4b9d9817a23f9a9cd95c as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/cdc9149df98a4b9d9817a23f9a9cd95c 2024-11-14T14:57:26,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741864_1040 (size=118898) 2024-11-14T14:57:26,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741864_1040 (size=118898) 2024-11-14T14:57:26,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/cdc9149df98a4b9d9817a23f9a9cd95c, entries=18, sequenceid=235, filesize=23.8 K 2024-11-14T14:57:26,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~18.91 KB/19368, heapSize ~20.48 KB/20976, currentSize=8.41 KB/8608 for c872993953da71467947c204b88318d7 in 25ms, sequenceid=235, compaction requested=false 2024-11-14T14:57:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:26,257 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/0ff1d1c5eae94dda8638d77c2437544a as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/0ff1d1c5eae94dda8638d77c2437544a 2024-11-14T14:57:26,262 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c872993953da71467947c204b88318d7/info of c872993953da71467947c204b88318d7 into 0ff1d1c5eae94dda8638d77c2437544a(size=116.1 K), total size for store is 139.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T14:57:26,262 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:26,262 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7., storeName=c872993953da71467947c204b88318d7/info, priority=13, startTime=1731596246230; duration=0sec 2024-11-14T14:57:26,262 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:57:26,262 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c872993953da71467947c204b88318d7:info 2024-11-14T14:57:26,600 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:26,600 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:27,600 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:27,600 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on c872993953da71467947c204b88318d7 2024-11-14T14:57:28,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c872993953da71467947c204b88318d7 1/1 column families, dataSize=9.46 KB heapSize=10.38 KB 2024-11-14T14:57:28,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/68f45e4b52f04717bd878adcd1dd1a66 is 1080, key is row0185/info:/1731596246232/Put/seqid=0 2024-11-14T14:57:28,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741865_1041 (size=14672) 2024-11-14T14:57:28,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741865_1041 (size=14672) 2024-11-14T14:57:28,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.46 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/68f45e4b52f04717bd878adcd1dd1a66 2024-11-14T14:57:28,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/68f45e4b52f04717bd878adcd1dd1a66 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/68f45e4b52f04717bd878adcd1dd1a66 2024-11-14T14:57:28,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/68f45e4b52f04717bd878adcd1dd1a66, entries=9, sequenceid=248, filesize=14.3 K 2024-11-14T14:57:28,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~9.46 KB/9684, heapSize ~10.36 KB/10608, currentSize=14.71 KB/15064 for c872993953da71467947c204b88318d7 in 26ms, sequenceid=248, compaction requested=true 2024-11-14T14:57:28,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:28,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c872993953da71467947c204b88318d7:info, priority=-2147483648, current under compaction store size is 1 2024-11-14T14:57:28,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:57:28,279 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T14:57:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on c872993953da71467947c204b88318d7 2024-11-14T14:57:28,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c872993953da71467947c204b88318d7 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-11-14T14:57:28,280 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 157964 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T14:57:28,281 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1541): c872993953da71467947c204b88318d7/info is initiating minor compaction (all files) 2024-11-14T14:57:28,281 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c872993953da71467947c204b88318d7/info in TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7. 2024-11-14T14:57:28,281 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/0ff1d1c5eae94dda8638d77c2437544a, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/cdc9149df98a4b9d9817a23f9a9cd95c, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/68f45e4b52f04717bd878adcd1dd1a66] into tmpdir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp, totalSize=154.3 K 2024-11-14T14:57:28,281 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ff1d1c5eae94dda8638d77c2437544a, keycount=105, bloomtype=ROW, size=116.1 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731596207716 2024-11-14T14:57:28,282 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdc9149df98a4b9d9817a23f9a9cd95c, keycount=18, bloomtype=ROW, size=23.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731596246202 2024-11-14T14:57:28,282 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68f45e4b52f04717bd878adcd1dd1a66, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731596246232 2024-11-14T14:57:28,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/8702300ab75e49de8b6e18f1e0257087 is 1080, key is row0194/info:/1731596248254/Put/seqid=0 2024-11-14T14:57:28,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741866_1042 (size=21165) 2024-11-14T14:57:28,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741866_1042 (size=21165) 2024-11-14T14:57:28,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/8702300ab75e49de8b6e18f1e0257087 2024-11-14T14:57:28,297 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c872993953da71467947c204b88318d7#info#compaction#82 average throughput is 33.86 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T14:57:28,298 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/9750c97b4f7f4c99a796da6704c056a7 is 1080, key is row0062/info:/1731596207716/Put/seqid=0 2024-11-14T14:57:28,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/8702300ab75e49de8b6e18f1e0257087 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/8702300ab75e49de8b6e18f1e0257087 2024-11-14T14:57:28,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/8702300ab75e49de8b6e18f1e0257087, entries=15, sequenceid=266, filesize=20.7 K 2024-11-14T14:57:28,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=11.56 KB/11836 for c872993953da71467947c204b88318d7 in 25ms, sequenceid=266, compaction requested=false 2024-11-14T14:57:28,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on c872993953da71467947c204b88318d7 2024-11-14T14:57:28,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c872993953da71467947c204b88318d7 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-11-14T14:57:28,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/f771624f20bd4b6d8924a1ca1fa8f338 is 1080, key is row0209/info:/1731596248281/Put/seqid=0 2024-11-14T14:57:28,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741867_1043 (size=148311) 2024-11-14T14:57:28,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741867_1043 (size=148311) 2024-11-14T14:57:28,322 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/9750c97b4f7f4c99a796da6704c056a7 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/9750c97b4f7f4c99a796da6704c056a7 2024-11-14T14:57:28,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741868_1044 (size=17918) 2024-11-14T14:57:28,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741868_1044 (size=17918) 2024-11-14T14:57:28,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/f771624f20bd4b6d8924a1ca1fa8f338 2024-11-14T14:57:28,328 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c872993953da71467947c204b88318d7/info of c872993953da71467947c204b88318d7 into 9750c97b4f7f4c99a796da6704c056a7(size=144.8 K), total size for store is 165.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T14:57:28,328 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:28,328 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7., storeName=c872993953da71467947c204b88318d7/info, priority=13, startTime=1731596248279; duration=0sec 2024-11-14T14:57:28,329 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:57:28,329 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c872993953da71467947c204b88318d7:info 2024-11-14T14:57:28,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/f771624f20bd4b6d8924a1ca1fa8f338 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/f771624f20bd4b6d8924a1ca1fa8f338 2024-11-14T14:57:28,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/f771624f20bd4b6d8924a1ca1fa8f338, entries=12, sequenceid=281, filesize=17.5 K 2024-11-14T14:57:28,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=4.20 KB/4304 for c872993953da71467947c204b88318d7 in 33ms, sequenceid=281, compaction requested=true 2024-11-14T14:57:28,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:28,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c872993953da71467947c204b88318d7:info, priority=-2147483648, current under compaction store size is 1 2024-11-14T14:57:28,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:57:28,339 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T14:57:28,340 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 187394 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T14:57:28,340 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1541): c872993953da71467947c204b88318d7/info is initiating minor compaction (all files) 2024-11-14T14:57:28,341 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c872993953da71467947c204b88318d7/info in TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7. 2024-11-14T14:57:28,341 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/9750c97b4f7f4c99a796da6704c056a7, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/8702300ab75e49de8b6e18f1e0257087, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/f771624f20bd4b6d8924a1ca1fa8f338] into tmpdir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp, totalSize=183.0 K 2024-11-14T14:57:28,341 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9750c97b4f7f4c99a796da6704c056a7, keycount=132, bloomtype=ROW, size=144.8 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731596207716 2024-11-14T14:57:28,341 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8702300ab75e49de8b6e18f1e0257087, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731596248254 2024-11-14T14:57:28,342 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting f771624f20bd4b6d8924a1ca1fa8f338, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731596248281 2024-11-14T14:57:28,353 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c872993953da71467947c204b88318d7#info#compaction#84 average throughput is 54.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T14:57:28,354 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/0723d31e10dd4fd8a9e98468f4c021a4 is 1080, key is row0062/info:/1731596207716/Put/seqid=0 2024-11-14T14:57:28,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741869_1045 (size=177548) 2024-11-14T14:57:28,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741869_1045 (size=177548) 2024-11-14T14:57:28,366 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/0723d31e10dd4fd8a9e98468f4c021a4 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/0723d31e10dd4fd8a9e98468f4c021a4 2024-11-14T14:57:28,372 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c872993953da71467947c204b88318d7/info of c872993953da71467947c204b88318d7 into 0723d31e10dd4fd8a9e98468f4c021a4(size=173.4 K), total size for store is 173.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T14:57:28,372 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:28,372 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7., storeName=c872993953da71467947c204b88318d7/info, priority=13, startTime=1731596248339; duration=0sec 2024-11-14T14:57:28,372 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:57:28,372 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c872993953da71467947c204b88318d7:info 2024-11-14T14:57:28,601 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:28,601 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:29,601 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:29,601 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on c872993953da71467947c204b88318d7 2024-11-14T14:57:30,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c872993953da71467947c204b88318d7 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-14T14:57:30,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/c6768f50a27a498f99a11aff5f5b9cdd is 1080, key is row0221/info:/1731596248307/Put/seqid=0 2024-11-14T14:57:30,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741870_1046 (size=12523) 2024-11-14T14:57:30,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741870_1046 (size=12523) 2024-11-14T14:57:30,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/c6768f50a27a498f99a11aff5f5b9cdd 2024-11-14T14:57:30,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/c6768f50a27a498f99a11aff5f5b9cdd as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/c6768f50a27a498f99a11aff5f5b9cdd 2024-11-14T14:57:30,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/c6768f50a27a498f99a11aff5f5b9cdd, entries=7, sequenceid=293, filesize=12.2 K 2024-11-14T14:57:30,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for c872993953da71467947c204b88318d7 in 23ms, sequenceid=293, compaction requested=false 2024-11-14T14:57:30,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:30,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36857 {}] regionserver.HRegion(8855): Flush requested on c872993953da71467947c204b88318d7 2024-11-14T14:57:30,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c872993953da71467947c204b88318d7 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-11-14T14:57:30,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/eeefce1de9464df5b7958606bb63540a is 1080, key is row0228/info:/1731596250320/Put/seqid=0 2024-11-14T14:57:30,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741871_1047 (size=21171) 2024-11-14T14:57:30,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741871_1047 (size=21171) 2024-11-14T14:57:30,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/eeefce1de9464df5b7958606bb63540a 2024-11-14T14:57:30,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/eeefce1de9464df5b7958606bb63540a as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/eeefce1de9464df5b7958606bb63540a 2024-11-14T14:57:30,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/eeefce1de9464df5b7958606bb63540a, entries=15, sequenceid=311, filesize=20.7 K 2024-11-14T14:57:30,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=14.71 KB/15064 for c872993953da71467947c204b88318d7 in 26ms, sequenceid=311, compaction requested=true 2024-11-14T14:57:30,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:30,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c872993953da71467947c204b88318d7:info, priority=-2147483648, current under compaction store size is 1 2024-11-14T14:57:30,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:57:30,371 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-14T14:57:30,372 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 211242 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-14T14:57:30,372 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1541): c872993953da71467947c204b88318d7/info is initiating minor compaction (all files) 2024-11-14T14:57:30,372 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c872993953da71467947c204b88318d7/info in TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7. 2024-11-14T14:57:30,372 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/0723d31e10dd4fd8a9e98468f4c021a4, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/c6768f50a27a498f99a11aff5f5b9cdd, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/eeefce1de9464df5b7958606bb63540a] into tmpdir=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp, totalSize=206.3 K 2024-11-14T14:57:30,372 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0723d31e10dd4fd8a9e98468f4c021a4, keycount=159, bloomtype=ROW, size=173.4 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731596207716 2024-11-14T14:57:30,373 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6768f50a27a498f99a11aff5f5b9cdd, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731596248307 2024-11-14T14:57:30,373 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeefce1de9464df5b7958606bb63540a, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731596250320 2024-11-14T14:57:30,384 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c872993953da71467947c204b88318d7#info#compaction#87 average throughput is 92.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T14:57:30,385 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/5a5d7fe9099e4ea5bcb635ab01c1bdf3 is 1080, key is row0062/info:/1731596207716/Put/seqid=0 2024-11-14T14:57:30,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741872_1048 (size=201392) 2024-11-14T14:57:30,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741872_1048 (size=201392) 2024-11-14T14:57:30,397 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/5a5d7fe9099e4ea5bcb635ab01c1bdf3 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/5a5d7fe9099e4ea5bcb635ab01c1bdf3 2024-11-14T14:57:30,403 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c872993953da71467947c204b88318d7/info of c872993953da71467947c204b88318d7 into 5a5d7fe9099e4ea5bcb635ab01c1bdf3(size=196.7 K), total size for store is 196.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T14:57:30,403 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:30,403 INFO [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7., storeName=c872993953da71467947c204b88318d7/info, priority=13, startTime=1731596250371; duration=0sec 2024-11-14T14:57:30,403 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-14T14:57:30,403 DEBUG [RS:0;da1c6afcd1f9:36857-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c872993953da71467947c204b88318d7:info 2024-11-14T14:57:30,602 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:30,602 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:31,602 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:31,602 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:32,367 INFO [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files 2024-11-14T14:57:32,367 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C36857%2C1731596194645.1731596252367 2024-11-14T14:57:32,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,374 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/WALs/da1c6afcd1f9,36857,1731596194645/da1c6afcd1f9%2C36857%2C1731596194645.1731596195037 with entries=308, filesize=307.10 KB; new WAL /user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/WALs/da1c6afcd1f9,36857,1731596194645/da1c6afcd1f9%2C36857%2C1731596194645.1731596252367 2024-11-14T14:57:32,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741833_1009 (size=314476) 2024-11-14T14:57:32,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741833_1009 (size=314476) 2024-11-14T14:57:32,378 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43735:43735),(127.0.0.1/127.0.0.1:34727:34727)] 2024-11-14T14:57:32,382 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for fafc242523dc305aa85bb931d0d24ef9: 2024-11-14T14:57:32,382 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=670 B heapSize=2.02 KB 2024-11-14T14:57:32,386 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/.tmp/info/063f3d198ec24d6fb93ba01a6695fb85 is 186, key is TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9./info:regioninfo/1731596210521/Put/seqid=0 2024-11-14T14:57:32,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741874_1050 (size=6153) 2024-11-14T14:57:32,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741874_1050 (size=6153) 2024-11-14T14:57:32,391 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=670 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/.tmp/info/063f3d198ec24d6fb93ba01a6695fb85 2024-11-14T14:57:32,396 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/.tmp/info/063f3d198ec24d6fb93ba01a6695fb85 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/info/063f3d198ec24d6fb93ba01a6695fb85 2024-11-14T14:57:32,401 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/info/063f3d198ec24d6fb93ba01a6695fb85, entries=5, sequenceid=21, filesize=6.0 K 2024-11-14T14:57:32,402 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~670 B/670, heapSize ~1.25 KB/1280, currentSize=0 B/0 for 1588230740 in 20ms, sequenceid=21, compaction requested=false 2024-11-14T14:57:32,402 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-14T14:57:32,402 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c872993953da71467947c204b88318d7 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-14T14:57:32,406 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/6a8a2b6b01ab441fa94cb71106cc0d40 is 1080, key is row0243/info:/1731596250345/Put/seqid=0 2024-11-14T14:57:32,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741875_1051 (size=20092) 2024-11-14T14:57:32,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741875_1051 (size=20092) 2024-11-14T14:57:32,411 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/6a8a2b6b01ab441fa94cb71106cc0d40 2024-11-14T14:57:32,416 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/.tmp/info/6a8a2b6b01ab441fa94cb71106cc0d40 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/6a8a2b6b01ab441fa94cb71106cc0d40 2024-11-14T14:57:32,420 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/6a8a2b6b01ab441fa94cb71106cc0d40, entries=14, sequenceid=329, filesize=19.6 K 2024-11-14T14:57:32,421 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=0 B/0 for c872993953da71467947c204b88318d7 in 19ms, sequenceid=329, compaction requested=false 2024-11-14T14:57:32,421 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c872993953da71467947c204b88318d7: 2024-11-14T14:57:32,421 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C36857%2C1731596194645.1731596252421 2024-11-14T14:57:32,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,426 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/WALs/da1c6afcd1f9,36857,1731596194645/da1c6afcd1f9%2C36857%2C1731596194645.1731596252367 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/WALs/da1c6afcd1f9,36857,1731596194645/da1c6afcd1f9%2C36857%2C1731596194645.1731596252421 2024-11-14T14:57:32,427 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34727:34727),(127.0.0.1/127.0.0.1:43735:43735)] 2024-11-14T14:57:32,427 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/WALs/da1c6afcd1f9,36857,1731596194645/da1c6afcd1f9%2C36857%2C1731596194645.1731596252367 is not closed yet, will try archiving it next time 2024-11-14T14:57:32,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741873_1049 (size=731) 2024-11-14T14:57:32,427 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/WALs/da1c6afcd1f9,36857,1731596194645/da1c6afcd1f9%2C36857%2C1731596194645.1731596195037 to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/oldWALs/da1c6afcd1f9%2C36857%2C1731596194645.1731596195037 2024-11-14T14:57:32,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741873_1049 (size=731) 2024-11-14T14:57:32,428 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T14:57:32,428 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/WALs/da1c6afcd1f9,36857,1731596194645/da1c6afcd1f9%2C36857%2C1731596194645.1731596252367 to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/oldWALs/da1c6afcd1f9%2C36857%2C1731596194645.1731596252367 2024-11-14T14:57:32,528 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T14:57:32,528 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:57:32,528 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:57:32,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:57:32,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:57:32,528 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T14:57:32,529 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T14:57:32,529 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=658906858, stopped=false 2024-11-14T14:57:32,529 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=da1c6afcd1f9,41179,1731596194593 2024-11-14T14:57:32,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:57:32,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:57:32,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:57:32,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:57:32,530 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:57:32,531 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:57:32,531 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:57:32,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:57:32,531 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'da1c6afcd1f9,36857,1731596194645' ***** 2024-11-14T14:57:32,531 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T14:57:32,531 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:57:32,531 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:57:32,531 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T14:57:32,531 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T14:57:32,532 INFO [RS:0;da1c6afcd1f9:36857 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T14:57:32,532 INFO [RS:0;da1c6afcd1f9:36857 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T14:57:32,532 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.HRegionServer(3091): Received CLOSE for fafc242523dc305aa85bb931d0d24ef9 2024-11-14T14:57:32,532 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.HRegionServer(3091): Received CLOSE for c872993953da71467947c204b88318d7 2024-11-14T14:57:32,532 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.HRegionServer(959): stopping server da1c6afcd1f9,36857,1731596194645 2024-11-14T14:57:32,532 INFO [RS:0;da1c6afcd1f9:36857 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:57:32,532 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fafc242523dc305aa85bb931d0d24ef9, disabling compactions & flushes 2024-11-14T14:57:32,532 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9. 2024-11-14T14:57:32,532 INFO [RS:0;da1c6afcd1f9:36857 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;da1c6afcd1f9:36857. 2024-11-14T14:57:32,532 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9. 2024-11-14T14:57:32,532 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9. after waiting 0 ms 2024-11-14T14:57:32,532 DEBUG [RS:0;da1c6afcd1f9:36857 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:57:32,532 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9. 2024-11-14T14:57:32,532 DEBUG [RS:0;da1c6afcd1f9:36857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:57:32,532 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T14:57:32,532 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T14:57:32,533 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T14:57:32,533 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T14:57:32,533 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-14T14:57:32,533 DEBUG [RS:0;da1c6afcd1f9:36857 {}] regionserver.HRegionServer(1325): Online Regions={fafc242523dc305aa85bb931d0d24ef9=TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9., 1588230740=hbase:meta,,1.1588230740, c872993953da71467947c204b88318d7=TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.} 2024-11-14T14:57:32,533 DEBUG [RS:0;da1c6afcd1f9:36857 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c872993953da71467947c204b88318d7, fafc242523dc305aa85bb931d0d24ef9 2024-11-14T14:57:32,533 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:57:32,533 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/fafc242523dc305aa85bb931d0d24ef9/info/704ee65e0b84460787443e1bc1ef2c05.8078e0e84354cb525a824451f0593eef->hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/704ee65e0b84460787443e1bc1ef2c05-bottom] to archive 2024-11-14T14:57:32,533 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:57:32,533 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:57:32,533 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:57:32,533 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:57:32,534 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T14:57:32,536 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/fafc242523dc305aa85bb931d0d24ef9/info/704ee65e0b84460787443e1bc1ef2c05.8078e0e84354cb525a824451f0593eef to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/fafc242523dc305aa85bb931d0d24ef9/info/704ee65e0b84460787443e1bc1ef2c05.8078e0e84354cb525a824451f0593eef 2024-11-14T14:57:32,536 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=da1c6afcd1f9:41179 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 16 more 2024-11-14T14:57:32,537 WARN [StoreCloser-TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [] 2024-11-14T14:57:32,538 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-14T14:57:32,539 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:57:32,539 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:57:32,539 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731596252533Running coprocessor pre-close hooks at 1731596252533Disabling compacts and flushes for region at 1731596252533Disabling writes for close at 1731596252533Writing region close event to WAL at 1731596252535 (+2 ms)Running coprocessor post-close hooks at 1731596252539 (+4 ms)Closed at 1731596252539 2024-11-14T14:57:32,539 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T14:57:32,540 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/fafc242523dc305aa85bb931d0d24ef9/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=121 2024-11-14T14:57:32,541 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9. 2024-11-14T14:57:32,541 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fafc242523dc305aa85bb931d0d24ef9: Waiting for close lock at 1731596252532Running coprocessor pre-close hooks at 1731596252532Disabling compacts and flushes for region at 1731596252532Disabling writes for close at 1731596252532Writing region close event to WAL at 1731596252537 (+5 ms)Running coprocessor post-close hooks at 1731596252541 (+4 ms)Closed at 1731596252541 2024-11-14T14:57:32,541 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1731596209803.fafc242523dc305aa85bb931d0d24ef9. 2024-11-14T14:57:32,541 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c872993953da71467947c204b88318d7, disabling compactions & flushes 2024-11-14T14:57:32,541 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7. 2024-11-14T14:57:32,541 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7. 2024-11-14T14:57:32,541 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7. after waiting 0 ms 2024-11-14T14:57:32,541 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7. 2024-11-14T14:57:32,541 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/704ee65e0b84460787443e1bc1ef2c05.8078e0e84354cb525a824451f0593eef->hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/8078e0e84354cb525a824451f0593eef/info/704ee65e0b84460787443e1bc1ef2c05-top, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/TestLogRolling-testLogRolling=8078e0e84354cb525a824451f0593eef-956cdf1b26ef41ae843e91a9fda9b942, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/62e4107d4f1c4357aa8e725e15efeb2d, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/TestLogRolling-testLogRolling=8078e0e84354cb525a824451f0593eef-71c5a5d0802f4ffc9f81d716fa53fdc6, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/0c3add6a228740ecb2c25c7fbce97d47, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/75afe4e1bda74ff8950fbb14702a5ad9, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/bc60517d583a4c1c9ab6db0a1a890c4a, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/12bea0b58b734e21bfb966763354eca2, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/6e41f61029954d8296be7606807aeba1, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/bfc0970175f9453fab0f0cbda3e3346f, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/bedcd5ba35d94366a94fd0c8e982108a, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/0ff1d1c5eae94dda8638d77c2437544a, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/3bd4ca95e9114c678c00f8d1bf37c8e0, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/cdc9149df98a4b9d9817a23f9a9cd95c, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/9750c97b4f7f4c99a796da6704c056a7, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/68f45e4b52f04717bd878adcd1dd1a66, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/8702300ab75e49de8b6e18f1e0257087, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/0723d31e10dd4fd8a9e98468f4c021a4, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/f771624f20bd4b6d8924a1ca1fa8f338, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/c6768f50a27a498f99a11aff5f5b9cdd, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/eeefce1de9464df5b7958606bb63540a] to archive 2024-11-14T14:57:32,542 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T14:57:32,543 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/704ee65e0b84460787443e1bc1ef2c05.8078e0e84354cb525a824451f0593eef to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/704ee65e0b84460787443e1bc1ef2c05.8078e0e84354cb525a824451f0593eef 2024-11-14T14:57:32,545 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/TestLogRolling-testLogRolling=8078e0e84354cb525a824451f0593eef-956cdf1b26ef41ae843e91a9fda9b942 to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/TestLogRolling-testLogRolling=8078e0e84354cb525a824451f0593eef-956cdf1b26ef41ae843e91a9fda9b942 2024-11-14T14:57:32,546 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/62e4107d4f1c4357aa8e725e15efeb2d to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/62e4107d4f1c4357aa8e725e15efeb2d 2024-11-14T14:57:32,547 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/TestLogRolling-testLogRolling=8078e0e84354cb525a824451f0593eef-71c5a5d0802f4ffc9f81d716fa53fdc6 to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/TestLogRolling-testLogRolling=8078e0e84354cb525a824451f0593eef-71c5a5d0802f4ffc9f81d716fa53fdc6 2024-11-14T14:57:32,548 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/0c3add6a228740ecb2c25c7fbce97d47 to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/0c3add6a228740ecb2c25c7fbce97d47 2024-11-14T14:57:32,549 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/75afe4e1bda74ff8950fbb14702a5ad9 to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/75afe4e1bda74ff8950fbb14702a5ad9 2024-11-14T14:57:32,550 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/bc60517d583a4c1c9ab6db0a1a890c4a to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/bc60517d583a4c1c9ab6db0a1a890c4a 2024-11-14T14:57:32,551 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/12bea0b58b734e21bfb966763354eca2 to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/12bea0b58b734e21bfb966763354eca2 2024-11-14T14:57:32,552 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/6e41f61029954d8296be7606807aeba1 to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/6e41f61029954d8296be7606807aeba1 2024-11-14T14:57:32,553 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/bfc0970175f9453fab0f0cbda3e3346f to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/bfc0970175f9453fab0f0cbda3e3346f 2024-11-14T14:57:32,554 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/bedcd5ba35d94366a94fd0c8e982108a to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/bedcd5ba35d94366a94fd0c8e982108a 2024-11-14T14:57:32,555 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/0ff1d1c5eae94dda8638d77c2437544a to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/0ff1d1c5eae94dda8638d77c2437544a 2024-11-14T14:57:32,555 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/3bd4ca95e9114c678c00f8d1bf37c8e0 to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/3bd4ca95e9114c678c00f8d1bf37c8e0 2024-11-14T14:57:32,556 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/cdc9149df98a4b9d9817a23f9a9cd95c to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/cdc9149df98a4b9d9817a23f9a9cd95c 2024-11-14T14:57:32,558 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/9750c97b4f7f4c99a796da6704c056a7 to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/9750c97b4f7f4c99a796da6704c056a7 2024-11-14T14:57:32,559 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/68f45e4b52f04717bd878adcd1dd1a66 to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/68f45e4b52f04717bd878adcd1dd1a66 2024-11-14T14:57:32,560 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/8702300ab75e49de8b6e18f1e0257087 to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/8702300ab75e49de8b6e18f1e0257087 2024-11-14T14:57:32,561 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/0723d31e10dd4fd8a9e98468f4c021a4 to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/0723d31e10dd4fd8a9e98468f4c021a4 2024-11-14T14:57:32,562 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/f771624f20bd4b6d8924a1ca1fa8f338 to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/f771624f20bd4b6d8924a1ca1fa8f338 2024-11-14T14:57:32,563 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/c6768f50a27a498f99a11aff5f5b9cdd to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/c6768f50a27a498f99a11aff5f5b9cdd 2024-11-14T14:57:32,564 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/eeefce1de9464df5b7958606bb63540a to hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/archive/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/info/eeefce1de9464df5b7958606bb63540a 2024-11-14T14:57:32,564 WARN [StoreCloser-TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [62e4107d4f1c4357aa8e725e15efeb2d=40830, 0c3add6a228740ecb2c25c7fbce97d47=12513, 75afe4e1bda74ff8950fbb14702a5ad9=71185, bc60517d583a4c1c9ab6db0a1a890c4a=27628, 12bea0b58b734e21bfb966763354eca2=12516, 6e41f61029954d8296be7606807aeba1=96154, bfc0970175f9453fab0f0cbda3e3346f=22238, bedcd5ba35d94366a94fd0c8e982108a=20078, 0ff1d1c5eae94dda8638d77c2437544a=118898, 3bd4ca95e9114c678c00f8d1bf37c8e0=12516, cdc9149df98a4b9d9817a23f9a9cd95c=24394, 9750c97b4f7f4c99a796da6704c056a7=148311, 68f45e4b52f04717bd878adcd1dd1a66=14672, 8702300ab75e49de8b6e18f1e0257087=21165, 0723d31e10dd4fd8a9e98468f4c021a4=177548, f771624f20bd4b6d8924a1ca1fa8f338=17918, c6768f50a27a498f99a11aff5f5b9cdd=12523, eeefce1de9464df5b7958606bb63540a=21171] 2024-11-14T14:57:32,568 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/data/default/TestLogRolling-testLogRolling/c872993953da71467947c204b88318d7/recovered.edits/332.seqid, newMaxSeqId=332, maxSeqId=121 2024-11-14T14:57:32,569 INFO [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7. 2024-11-14T14:57:32,569 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c872993953da71467947c204b88318d7: Waiting for close lock at 1731596252541Running coprocessor pre-close hooks at 1731596252541Disabling compacts and flushes for region at 1731596252541Disabling writes for close at 1731596252541Writing region close event to WAL at 1731596252565 (+24 ms)Running coprocessor post-close hooks at 1731596252569 (+4 ms)Closed at 1731596252569 2024-11-14T14:57:32,569 DEBUG [RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1731596209803.c872993953da71467947c204b88318d7. 2024-11-14T14:57:32,603 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:32,603 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:32,733 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.HRegionServer(976): stopping server da1c6afcd1f9,36857,1731596194645; all regions closed. 2024-11-14T14:57:32,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741834_1010 (size=8107) 2024-11-14T14:57:32,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741834_1010 (size=8107) 2024-11-14T14:57:32,738 DEBUG [RS:0;da1c6afcd1f9:36857 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/oldWALs 2024-11-14T14:57:32,739 INFO [RS:0;da1c6afcd1f9:36857 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog da1c6afcd1f9%2C36857%2C1731596194645.meta:.meta(num 1731596195412) 2024-11-14T14:57:32,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741876_1052 (size=778) 2024-11-14T14:57:32,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741876_1052 (size=778) 2024-11-14T14:57:32,743 DEBUG [RS:0;da1c6afcd1f9:36857 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/oldWALs 2024-11-14T14:57:32,743 INFO [RS:0;da1c6afcd1f9:36857 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog da1c6afcd1f9%2C36857%2C1731596194645:(num 1731596252421) 2024-11-14T14:57:32,743 DEBUG [RS:0;da1c6afcd1f9:36857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:57:32,743 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:57:32,743 INFO [RS:0;da1c6afcd1f9:36857 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:57:32,743 INFO [RS:0;da1c6afcd1f9:36857 {}] hbase.ChoreService(370): Chore service for: regionserver/da1c6afcd1f9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T14:57:32,744 INFO [RS:0;da1c6afcd1f9:36857 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:57:32,744 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:57:32,744 INFO [RS:0;da1c6afcd1f9:36857 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36857 2024-11-14T14:57:32,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/da1c6afcd1f9,36857,1731596194645 2024-11-14T14:57:32,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:57:32,746 INFO [RS:0;da1c6afcd1f9:36857 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:57:32,747 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [da1c6afcd1f9,36857,1731596194645] 2024-11-14T14:57:32,749 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/da1c6afcd1f9,36857,1731596194645 already deleted, retry=false 2024-11-14T14:57:32,749 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; da1c6afcd1f9,36857,1731596194645 expired; onlineServers=0 2024-11-14T14:57:32,749 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'da1c6afcd1f9,41179,1731596194593' ***** 2024-11-14T14:57:32,749 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T14:57:32,749 INFO [M:0;da1c6afcd1f9:41179 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:57:32,749 INFO [M:0;da1c6afcd1f9:41179 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:57:32,749 DEBUG [M:0;da1c6afcd1f9:41179 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T14:57:32,749 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T14:57:32,749 DEBUG [M:0;da1c6afcd1f9:41179 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T14:57:32,749 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731596194810 {}] cleaner.HFileCleaner(306): Exit Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731596194810,5,FailOnTimeoutGroup] 2024-11-14T14:57:32,749 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731596194810 {}] cleaner.HFileCleaner(306): Exit Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731596194810,5,FailOnTimeoutGroup] 2024-11-14T14:57:32,749 INFO [M:0;da1c6afcd1f9:41179 {}] hbase.ChoreService(370): Chore service for: master/da1c6afcd1f9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T14:57:32,750 INFO [M:0;da1c6afcd1f9:41179 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:57:32,750 DEBUG [M:0;da1c6afcd1f9:41179 {}] master.HMaster(1795): Stopping service threads 2024-11-14T14:57:32,750 INFO [M:0;da1c6afcd1f9:41179 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T14:57:32,750 INFO [M:0;da1c6afcd1f9:41179 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:57:32,750 INFO [M:0;da1c6afcd1f9:41179 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:57:32,750 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T14:57:32,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T14:57:32,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:57:32,751 DEBUG [M:0;da1c6afcd1f9:41179 {}] zookeeper.ZKUtil(347): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T14:57:32,751 WARN [M:0;da1c6afcd1f9:41179 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T14:57:32,751 INFO [M:0;da1c6afcd1f9:41179 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/.lastflushedseqids 2024-11-14T14:57:32,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741877_1053 (size=228) 2024-11-14T14:57:32,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741877_1053 (size=228) 2024-11-14T14:57:32,757 INFO [M:0;da1c6afcd1f9:41179 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:57:32,757 INFO [M:0;da1c6afcd1f9:41179 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T14:57:32,757 DEBUG [M:0;da1c6afcd1f9:41179 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:57:32,757 INFO [M:0;da1c6afcd1f9:41179 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:57:32,757 DEBUG [M:0;da1c6afcd1f9:41179 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:57:32,757 DEBUG [M:0;da1c6afcd1f9:41179 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:57:32,758 DEBUG [M:0;da1c6afcd1f9:41179 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:57:32,758 INFO [M:0;da1c6afcd1f9:41179 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.43 KB heapSize=63.36 KB 2024-11-14T14:57:32,778 DEBUG [M:0;da1c6afcd1f9:41179 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1279f145e5534f5ba8c03b8b49a766d7 is 82, key is hbase:meta,,1/info:regioninfo/1731596195442/Put/seqid=0 2024-11-14T14:57:32,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741878_1054 (size=5672) 2024-11-14T14:57:32,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741878_1054 (size=5672) 2024-11-14T14:57:32,786 INFO [M:0;da1c6afcd1f9:41179 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1279f145e5534f5ba8c03b8b49a766d7 2024-11-14T14:57:32,808 DEBUG [M:0;da1c6afcd1f9:41179 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/beba334c2bc44857992fd8f33ebb3fa9 is 751, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731596195948/Put/seqid=0 2024-11-14T14:57:32,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741879_1055 (size=7091) 2024-11-14T14:57:32,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741879_1055 (size=7091) 2024-11-14T14:57:32,821 INFO [M:0;da1c6afcd1f9:41179 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.83 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/beba334c2bc44857992fd8f33ebb3fa9 2024-11-14T14:57:32,826 INFO [M:0;da1c6afcd1f9:41179 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for beba334c2bc44857992fd8f33ebb3fa9 2024-11-14T14:57:32,847 DEBUG [M:0;da1c6afcd1f9:41179 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/061b38ae1dc6410e83d358773d0366ac is 69, key is da1c6afcd1f9,36857,1731596194645/rs:state/1731596194881/Put/seqid=0 2024-11-14T14:57:32,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:57:32,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36857-0x101a748887a0001, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:57:32,848 INFO [RS:0;da1c6afcd1f9:36857 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:57:32,848 INFO [RS:0;da1c6afcd1f9:36857 {}] regionserver.HRegionServer(1031): Exiting; stopping=da1c6afcd1f9,36857,1731596194645; zookeeper connection closed. 2024-11-14T14:57:32,848 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@54ff1272 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@54ff1272 2024-11-14T14:57:32,848 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T14:57:32,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741880_1056 (size=5156) 2024-11-14T14:57:32,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741880_1056 (size=5156) 2024-11-14T14:57:32,854 INFO [M:0;da1c6afcd1f9:41179 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/061b38ae1dc6410e83d358773d0366ac 2024-11-14T14:57:32,873 DEBUG [M:0;da1c6afcd1f9:41179 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e3cefa42d5614d0398648a0c5382dda3 is 52, key is load_balancer_on/state:d/1731596195571/Put/seqid=0 2024-11-14T14:57:32,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741881_1057 (size=5056) 2024-11-14T14:57:32,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741881_1057 (size=5056) 2024-11-14T14:57:32,878 INFO [M:0;da1c6afcd1f9:41179 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e3cefa42d5614d0398648a0c5382dda3 2024-11-14T14:57:32,884 DEBUG [M:0;da1c6afcd1f9:41179 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1279f145e5534f5ba8c03b8b49a766d7 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1279f145e5534f5ba8c03b8b49a766d7 2024-11-14T14:57:32,888 INFO [M:0;da1c6afcd1f9:41179 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1279f145e5534f5ba8c03b8b49a766d7, entries=8, sequenceid=125, filesize=5.5 K 2024-11-14T14:57:32,890 DEBUG [M:0;da1c6afcd1f9:41179 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/beba334c2bc44857992fd8f33ebb3fa9 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/beba334c2bc44857992fd8f33ebb3fa9 2024-11-14T14:57:32,895 INFO [M:0;da1c6afcd1f9:41179 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for beba334c2bc44857992fd8f33ebb3fa9 2024-11-14T14:57:32,895 INFO [M:0;da1c6afcd1f9:41179 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/beba334c2bc44857992fd8f33ebb3fa9, entries=13, sequenceid=125, filesize=6.9 K 2024-11-14T14:57:32,896 DEBUG [M:0;da1c6afcd1f9:41179 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/061b38ae1dc6410e83d358773d0366ac as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/061b38ae1dc6410e83d358773d0366ac 2024-11-14T14:57:32,901 INFO [M:0;da1c6afcd1f9:41179 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/061b38ae1dc6410e83d358773d0366ac, entries=1, sequenceid=125, filesize=5.0 K 2024-11-14T14:57:32,901 INFO [regionserver/da1c6afcd1f9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:57:32,901 DEBUG [M:0;da1c6afcd1f9:41179 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e3cefa42d5614d0398648a0c5382dda3 as hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e3cefa42d5614d0398648a0c5382dda3 2024-11-14T14:57:32,906 INFO [M:0;da1c6afcd1f9:41179 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41455/user/jenkins/test-data/c3ea08ea-0dcb-f33f-f7f1-965459a2e4d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e3cefa42d5614d0398648a0c5382dda3, entries=1, sequenceid=125, filesize=4.9 K 2024-11-14T14:57:32,907 INFO [M:0;da1c6afcd1f9:41179 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.30 KB/64816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 149ms, sequenceid=125, compaction requested=false 2024-11-14T14:57:32,921 INFO [M:0;da1c6afcd1f9:41179 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:57:32,921 DEBUG [M:0;da1c6afcd1f9:41179 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731596252757Disabling compacts and flushes for region at 1731596252757Disabling writes for close at 1731596252757Obtaining lock to block concurrent updates at 1731596252758 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731596252758Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52663, getHeapSize=64816, getOffHeapSize=0, getCellsCount=148 at 1731596252758Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731596252759 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731596252759Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731596252778 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731596252778Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731596252791 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731596252807 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731596252807Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731596252826 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731596252847 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731596252847Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731596252858 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731596252873 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731596252873Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7beac258: reopening flushed file at 1731596252883 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d5fe161: reopening flushed file at 1731596252889 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59139cee: reopening flushed file at 1731596252895 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18c8b679: reopening flushed file at 1731596252901 (+6 ms)Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.30 KB/64816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 149ms, sequenceid=125, compaction requested=false at 1731596252907 (+6 ms)Writing region close event to WAL at 1731596252921 (+14 ms)Closed at 1731596252921 2024-11-14T14:57:32,921 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:32,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33753 is added to blk_1073741830_1006 (size=61332) 2024-11-14T14:57:32,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35037 is added to blk_1073741830_1006 (size=61332) 2024-11-14T14:57:32,926 INFO [M:0;da1c6afcd1f9:41179 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T14:57:32,926 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:57:32,926 INFO [M:0;da1c6afcd1f9:41179 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41179 2024-11-14T14:57:32,926 INFO [M:0;da1c6afcd1f9:41179 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:57:33,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:57:33,029 INFO [M:0;da1c6afcd1f9:41179 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:57:33,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41179-0x101a748887a0000, quorum=127.0.0.1:63286, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:57:33,031 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2f932cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:57:33,032 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5ff7780b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:57:33,032 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:57:33,032 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2735da07{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:57:33,032 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@16ccf5f4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/hadoop.log.dir/,STOPPED} 2024-11-14T14:57:33,034 WARN [BP-906651502-172.17.0.2-1731596193805 heartbeating to localhost/127.0.0.1:41455 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:57:33,034 WARN [BP-906651502-172.17.0.2-1731596193805 heartbeating to localhost/127.0.0.1:41455 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-906651502-172.17.0.2-1731596193805 (Datanode Uuid df7bd351-504e-49a5-9f21-62de66989bf3) service to localhost/127.0.0.1:41455 2024-11-14T14:57:33,034 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:57:33,034 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:57:33,035 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/cluster_82567f71-e0c9-9c1a-8289-309cd8e137b7/data/data3/current/BP-906651502-172.17.0.2-1731596193805 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:57:33,035 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/cluster_82567f71-e0c9-9c1a-8289-309cd8e137b7/data/data4/current/BP-906651502-172.17.0.2-1731596193805 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:57:33,035 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:57:33,038 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4a67ff9c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:57:33,038 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3bd9a438{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:57:33,038 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:57:33,039 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6f8f17a1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:57:33,039 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@201bdbf9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/hadoop.log.dir/,STOPPED} 2024-11-14T14:57:33,040 WARN [BP-906651502-172.17.0.2-1731596193805 heartbeating to localhost/127.0.0.1:41455 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:57:33,041 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:57:33,041 WARN [BP-906651502-172.17.0.2-1731596193805 heartbeating to localhost/127.0.0.1:41455 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-906651502-172.17.0.2-1731596193805 (Datanode Uuid d5faaece-58bb-47a3-8d08-d5165224d771) service to localhost/127.0.0.1:41455 2024-11-14T14:57:33,041 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:57:33,041 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/cluster_82567f71-e0c9-9c1a-8289-309cd8e137b7/data/data1/current/BP-906651502-172.17.0.2-1731596193805 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:57:33,041 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/cluster_82567f71-e0c9-9c1a-8289-309cd8e137b7/data/data2/current/BP-906651502-172.17.0.2-1731596193805 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:57:33,041 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:57:33,049 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6e8e6cbc{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T14:57:33,049 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@188ddc10{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:57:33,049 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:57:33,049 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@52d230c9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:57:33,049 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@110e33ce{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/hadoop.log.dir/,STOPPED} 2024-11-14T14:57:33,057 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T14:57:33,083 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-14T14:57:33,093 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=232 (was 208) Potentially hanging thread: HMaster-EventLoopGroup-14-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41455 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:41455 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41455 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:41455 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41455 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-40-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41455 from jenkins.hfs.6 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41455 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41455 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=518 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=173 (was 180), ProcessCount=11 (was 11), AvailableMemoryMB=8502 (was 7252) - AvailableMemoryMB LEAK? - 2024-11-14T14:57:33,101 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=232, OpenFileDescriptor=518, MaxFileDescriptor=1048576, SystemLoadAverage=173, ProcessCount=11, AvailableMemoryMB=8501 2024-11-14T14:57:33,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T14:57:33,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/hadoop.log.dir so I do NOT create it in target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184 2024-11-14T14:57:33,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/49b49640-93db-6ffb-db6a-039b9dfabbf4/hadoop.tmp.dir so I do NOT create it in target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184 2024-11-14T14:57:33,102 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/cluster_0e145edd-8855-07ba-125d-fb4f19b1c6fa, deleteOnExit=true 2024-11-14T14:57:33,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T14:57:33,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/test.cache.data in system properties and HBase conf 2024-11-14T14:57:33,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T14:57:33,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/hadoop.log.dir in system properties and HBase conf 2024-11-14T14:57:33,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T14:57:33,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T14:57:33,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T14:57:33,103 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T14:57:33,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T14:57:33,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T14:57:33,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T14:57:33,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T14:57:33,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T14:57:33,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T14:57:33,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T14:57:33,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T14:57:33,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T14:57:33,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/nfs.dump.dir in system properties and HBase conf 2024-11-14T14:57:33,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/java.io.tmpdir in system properties and HBase conf 2024-11-14T14:57:33,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T14:57:33,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T14:57:33,103 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T14:57:33,116 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-14T14:57:33,240 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:57:33,244 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:57:33,248 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:57:33,248 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:57:33,248 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T14:57:33,249 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:57:33,252 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@30964c14{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:57:33,252 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@442ec576{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:57:33,413 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1b488929{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/java.io.tmpdir/jetty-localhost-39149-hadoop-hdfs-3_4_1-tests_jar-_-any-12712073782606875182/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T14:57:33,413 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@54644e01{HTTP/1.1, (http/1.1)}{localhost:39149} 2024-11-14T14:57:33,413 INFO [Time-limited test {}] server.Server(415): Started @299035ms 2024-11-14T14:57:33,432 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-14T14:57:33,511 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:57:33,515 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:57:33,521 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:57:33,521 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:57:33,521 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T14:57:33,522 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@677f535e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:57:33,522 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@35a6030c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:57:33,603 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:33,603 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:33,652 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3698cd86{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/java.io.tmpdir/jetty-localhost-44477-hadoop-hdfs-3_4_1-tests_jar-_-any-1483782826812171004/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:57:33,652 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@418bce8c{HTTP/1.1, (http/1.1)}{localhost:44477} 2024-11-14T14:57:33,652 INFO [Time-limited test {}] server.Server(415): Started @299274ms 2024-11-14T14:57:33,653 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:57:33,699 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T14:57:33,703 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T14:57:33,704 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T14:57:33,704 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T14:57:33,705 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T14:57:33,705 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@231297c7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/hadoop.log.dir/,AVAILABLE} 2024-11-14T14:57:33,706 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@536a5c19{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T14:57:33,755 WARN [Thread-2470 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/cluster_0e145edd-8855-07ba-125d-fb4f19b1c6fa/data/data1/current/BP-1130588054-172.17.0.2-1731596253125/current, will proceed with Du for space computation calculation, 2024-11-14T14:57:33,756 WARN [Thread-2471 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/cluster_0e145edd-8855-07ba-125d-fb4f19b1c6fa/data/data2/current/BP-1130588054-172.17.0.2-1731596253125/current, will proceed with Du for space computation calculation, 2024-11-14T14:57:33,786 WARN [Thread-2449 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:57:33,790 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x740fdc625d797f06 with lease ID 0x7e4fd21af6657081: Processing first storage report for DS-2ba03ec2-8178-468c-932b-966045614e4a from datanode DatanodeRegistration(127.0.0.1:37481, datanodeUuid=d8011282-0532-4b70-b48e-001dd156073c, infoPort=39353, infoSecurePort=0, ipcPort=45961, storageInfo=lv=-57;cid=testClusterID;nsid=198209492;c=1731596253125) 2024-11-14T14:57:33,790 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x740fdc625d797f06 with lease ID 0x7e4fd21af6657081: from storage DS-2ba03ec2-8178-468c-932b-966045614e4a node DatanodeRegistration(127.0.0.1:37481, datanodeUuid=d8011282-0532-4b70-b48e-001dd156073c, infoPort=39353, infoSecurePort=0, ipcPort=45961, storageInfo=lv=-57;cid=testClusterID;nsid=198209492;c=1731596253125), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:57:33,790 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x740fdc625d797f06 with lease ID 0x7e4fd21af6657081: Processing first storage report for DS-3cefd805-c5b7-4af3-860a-be001e704e4c from datanode DatanodeRegistration(127.0.0.1:37481, datanodeUuid=d8011282-0532-4b70-b48e-001dd156073c, infoPort=39353, infoSecurePort=0, ipcPort=45961, storageInfo=lv=-57;cid=testClusterID;nsid=198209492;c=1731596253125) 2024-11-14T14:57:33,790 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x740fdc625d797f06 with lease ID 0x7e4fd21af6657081: from storage DS-3cefd805-c5b7-4af3-860a-be001e704e4c node DatanodeRegistration(127.0.0.1:37481, datanodeUuid=d8011282-0532-4b70-b48e-001dd156073c, infoPort=39353, infoSecurePort=0, ipcPort=45961, storageInfo=lv=-57;cid=testClusterID;nsid=198209492;c=1731596253125), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T14:57:33,849 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d78da71{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/java.io.tmpdir/jetty-localhost-43205-hadoop-hdfs-3_4_1-tests_jar-_-any-9869271465792336237/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:57:33,849 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@39d5f486{HTTP/1.1, (http/1.1)}{localhost:43205} 2024-11-14T14:57:33,849 INFO [Time-limited test {}] server.Server(415): Started @299471ms 2024-11-14T14:57:33,850 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T14:57:33,948 WARN [Thread-2496 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/cluster_0e145edd-8855-07ba-125d-fb4f19b1c6fa/data/data3/current/BP-1130588054-172.17.0.2-1731596253125/current, will proceed with Du for space computation calculation, 2024-11-14T14:57:33,949 WARN [Thread-2497 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/cluster_0e145edd-8855-07ba-125d-fb4f19b1c6fa/data/data4/current/BP-1130588054-172.17.0.2-1731596253125/current, will proceed with Du for space computation calculation, 2024-11-14T14:57:33,970 WARN [Thread-2485 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T14:57:33,972 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8454d16a316ebe38 with lease ID 0x7e4fd21af6657082: Processing first storage report for DS-2c3c1b0b-07bf-4d79-a972-4ebf81bbef1f from datanode DatanodeRegistration(127.0.0.1:44669, datanodeUuid=732513ac-b026-4d46-8f04-ff2b213483cf, infoPort=39351, infoSecurePort=0, ipcPort=40975, storageInfo=lv=-57;cid=testClusterID;nsid=198209492;c=1731596253125) 2024-11-14T14:57:33,972 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8454d16a316ebe38 with lease ID 0x7e4fd21af6657082: from storage DS-2c3c1b0b-07bf-4d79-a972-4ebf81bbef1f node DatanodeRegistration(127.0.0.1:44669, datanodeUuid=732513ac-b026-4d46-8f04-ff2b213483cf, infoPort=39351, infoSecurePort=0, ipcPort=40975, storageInfo=lv=-57;cid=testClusterID;nsid=198209492;c=1731596253125), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:57:33,972 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8454d16a316ebe38 with lease ID 0x7e4fd21af6657082: Processing first storage report for DS-a594bb3e-35a0-4137-8478-08a6edceb785 from datanode DatanodeRegistration(127.0.0.1:44669, datanodeUuid=732513ac-b026-4d46-8f04-ff2b213483cf, infoPort=39351, infoSecurePort=0, ipcPort=40975, storageInfo=lv=-57;cid=testClusterID;nsid=198209492;c=1731596253125) 2024-11-14T14:57:33,972 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8454d16a316ebe38 with lease ID 0x7e4fd21af6657082: from storage DS-a594bb3e-35a0-4137-8478-08a6edceb785 node DatanodeRegistration(127.0.0.1:44669, datanodeUuid=732513ac-b026-4d46-8f04-ff2b213483cf, infoPort=39351, infoSecurePort=0, ipcPort=40975, storageInfo=lv=-57;cid=testClusterID;nsid=198209492;c=1731596253125), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T14:57:33,985 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184 2024-11-14T14:57:33,992 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/cluster_0e145edd-8855-07ba-125d-fb4f19b1c6fa/zookeeper_0, clientPort=50420, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/cluster_0e145edd-8855-07ba-125d-fb4f19b1c6fa/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/cluster_0e145edd-8855-07ba-125d-fb4f19b1c6fa/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T14:57:33,993 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50420 2024-11-14T14:57:33,993 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:57:33,994 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:57:34,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37481 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:57:34,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44669 is added to blk_1073741825_1001 (size=7) 2024-11-14T14:57:34,008 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4 with version=8 2024-11-14T14:57:34,008 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:37399/user/jenkins/test-data/446af988-3ca5-6bd8-4c7f-cfcfde548c9a/hbase-staging 2024-11-14T14:57:34,010 INFO [Time-limited test {}] client.ConnectionUtils(128): master/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:57:34,010 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:57:34,010 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:57:34,010 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:57:34,010 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:57:34,010 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:57:34,010 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T14:57:34,010 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:57:34,012 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46081 2024-11-14T14:57:34,014 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46081 connecting to ZooKeeper ensemble=127.0.0.1:50420 2024-11-14T14:57:34,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:460810x0, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:57:34,028 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46081-0x101a749708c0000 connected 2024-11-14T14:57:34,057 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:57:34,058 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:57:34,060 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:57:34,060 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4, hbase.cluster.distributed=false 2024-11-14T14:57:34,062 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:57:34,067 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46081 2024-11-14T14:57:34,067 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46081 2024-11-14T14:57:34,067 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46081 2024-11-14T14:57:34,067 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46081 2024-11-14T14:57:34,067 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46081 2024-11-14T14:57:34,082 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/da1c6afcd1f9:0 server-side Connection retries=45 2024-11-14T14:57:34,082 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:57:34,082 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T14:57:34,082 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T14:57:34,082 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T14:57:34,082 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T14:57:34,082 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T14:57:34,082 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T14:57:34,083 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36413 2024-11-14T14:57:34,084 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36413 connecting to ZooKeeper ensemble=127.0.0.1:50420 2024-11-14T14:57:34,084 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:57:34,086 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:57:34,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:364130x0, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T14:57:34,090 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36413-0x101a749708c0001 connected 2024-11-14T14:57:34,090 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:57:34,090 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T14:57:34,091 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T14:57:34,091 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T14:57:34,092 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T14:57:34,096 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36413 2024-11-14T14:57:34,097 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36413 2024-11-14T14:57:34,097 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36413 2024-11-14T14:57:34,098 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36413 2024-11-14T14:57:34,099 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36413 2024-11-14T14:57:34,111 DEBUG [M:0;da1c6afcd1f9:46081 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;da1c6afcd1f9:46081 2024-11-14T14:57:34,111 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/da1c6afcd1f9,46081,1731596254010 2024-11-14T14:57:34,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:57:34,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:57:34,113 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/da1c6afcd1f9,46081,1731596254010 2024-11-14T14:57:34,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T14:57:34,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:57:34,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:57:34,115 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T14:57:34,115 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/da1c6afcd1f9,46081,1731596254010 from backup master directory 2024-11-14T14:57:34,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/da1c6afcd1f9,46081,1731596254010 2024-11-14T14:57:34,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:57:34,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T14:57:34,116 WARN [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:57:34,116 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=da1c6afcd1f9,46081,1731596254010 2024-11-14T14:57:34,121 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/hbase.id] with ID: 5edeb74e-79f3-471c-a397-1663b0e2c073 2024-11-14T14:57:34,121 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/.tmp/hbase.id 2024-11-14T14:57:34,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37481 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:57:34,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44669 is added to blk_1073741826_1002 (size=42) 2024-11-14T14:57:34,132 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/.tmp/hbase.id]:[hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/hbase.id] 2024-11-14T14:57:34,144 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:57:34,145 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T14:57:34,146 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-14T14:57:34,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:57:34,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:57:34,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37481 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:57:34,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44669 is added to blk_1073741827_1003 (size=196) 2024-11-14T14:57:34,175 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T14:57:34,176 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T14:57:34,176 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:57:34,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44669 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:57:34,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37481 is added to blk_1073741828_1004 (size=1189) 2024-11-14T14:57:34,184 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store 2024-11-14T14:57:34,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44669 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:57:34,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37481 is added to blk_1073741829_1005 (size=34) 2024-11-14T14:57:34,190 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:57:34,190 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:57:34,190 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:57:34,190 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:57:34,190 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:57:34,190 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:57:34,190 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:57:34,190 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731596254190Disabling compacts and flushes for region at 1731596254190Disabling writes for close at 1731596254190Writing region close event to WAL at 1731596254190Closed at 1731596254190 2024-11-14T14:57:34,191 WARN [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/.initializing 2024-11-14T14:57:34,191 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/WALs/da1c6afcd1f9,46081,1731596254010 2024-11-14T14:57:34,194 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C46081%2C1731596254010, suffix=, logDir=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/WALs/da1c6afcd1f9,46081,1731596254010, archiveDir=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/oldWALs, maxLogs=10 2024-11-14T14:57:34,194 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C46081%2C1731596254010.1731596254194 2024-11-14T14:57:34,198 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/WALs/da1c6afcd1f9,46081,1731596254010/da1c6afcd1f9%2C46081%2C1731596254010.1731596254194 2024-11-14T14:57:34,199 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39353:39353),(127.0.0.1/127.0.0.1:39351:39351)] 2024-11-14T14:57:34,199 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:57:34,200 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:57:34,200 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:57:34,200 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:57:34,201 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:57:34,202 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T14:57:34,202 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:57:34,203 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:57:34,203 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:57:34,204 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T14:57:34,204 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:57:34,205 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:57:34,205 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:57:34,206 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T14:57:34,206 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:57:34,207 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:57:34,207 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:57:34,208 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T14:57:34,208 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:57:34,208 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T14:57:34,208 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:57:34,209 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:57:34,209 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:57:34,210 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:57:34,211 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:57:34,211 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T14:57:34,212 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T14:57:34,214 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:57:34,214 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=767831, jitterRate=-0.023652777075767517}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T14:57:34,215 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731596254200Initializing all the Stores at 1731596254201 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596254201Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596254201Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596254201Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596254201Cleaning up temporary data from old regions at 1731596254211 (+10 ms)Region opened successfully at 1731596254215 (+4 ms) 2024-11-14T14:57:34,215 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T14:57:34,218 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1831da4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:57:34,219 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T14:57:34,219 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T14:57:34,219 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T14:57:34,219 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T14:57:34,220 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-14T14:57:34,220 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-14T14:57:34,220 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T14:57:34,222 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T14:57:34,223 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T14:57:34,224 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T14:57:34,225 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T14:57:34,225 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T14:57:34,226 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T14:57:34,227 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T14:57:34,228 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T14:57:34,229 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T14:57:34,230 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T14:57:34,231 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T14:57:34,233 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T14:57:34,234 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T14:57:34,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:57:34,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T14:57:34,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:57:34,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:57:34,237 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=da1c6afcd1f9,46081,1731596254010, sessionid=0x101a749708c0000, setting cluster-up flag (Was=false) 2024-11-14T14:57:34,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:57:34,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:57:34,244 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T14:57:34,245 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da1c6afcd1f9,46081,1731596254010 2024-11-14T14:57:34,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:57:34,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:57:34,254 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T14:57:34,255 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=da1c6afcd1f9,46081,1731596254010 2024-11-14T14:57:34,256 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T14:57:34,258 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T14:57:34,258 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T14:57:34,258 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T14:57:34,258 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: da1c6afcd1f9,46081,1731596254010 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T14:57:34,260 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:57:34,260 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:57:34,260 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:57:34,260 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=5, maxPoolSize=5 2024-11-14T14:57:34,260 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/da1c6afcd1f9:0, corePoolSize=10, maxPoolSize=10 2024-11-14T14:57:34,260 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:57:34,260 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:57:34,260 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:57:34,261 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731596284261 2024-11-14T14:57:34,261 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T14:57:34,261 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T14:57:34,261 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:57:34,261 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T14:57:34,261 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T14:57:34,261 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T14:57:34,261 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T14:57:34,261 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T14:57:34,262 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,262 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:57:34,263 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T14:57:34,263 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T14:57:34,263 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T14:57:34,263 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T14:57:34,264 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T14:57:34,264 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T14:57:34,264 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731596254264,5,FailOnTimeoutGroup] 2024-11-14T14:57:34,271 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731596254264,5,FailOnTimeoutGroup] 2024-11-14T14:57:34,271 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,271 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T14:57:34,271 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,271 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44669 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:57:34,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37481 is added to blk_1073741831_1007 (size=1321) 2024-11-14T14:57:34,278 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T14:57:34,278 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4 2024-11-14T14:57:34,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44669 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:57:34,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37481 is added to blk_1073741832_1008 (size=32) 2024-11-14T14:57:34,284 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:57:34,285 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:57:34,286 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:57:34,286 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:57:34,287 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:57:34,287 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:57:34,288 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:57:34,288 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:57:34,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:57:34,289 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:57:34,290 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:57:34,290 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:57:34,290 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:57:34,290 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:57:34,291 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:57:34,291 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:57:34,292 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:57:34,292 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:57:34,292 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/data/hbase/meta/1588230740 2024-11-14T14:57:34,293 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/data/hbase/meta/1588230740 2024-11-14T14:57:34,294 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:57:34,294 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:57:34,295 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T14:57:34,296 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:57:34,297 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T14:57:34,298 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=859134, jitterRate=0.0924464613199234}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T14:57:34,299 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731596254284Initializing all the Stores at 1731596254285 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596254285Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596254285Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596254285Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596254285Cleaning up temporary data from old regions at 1731596254294 (+9 ms)Region opened successfully at 1731596254299 (+5 ms) 2024-11-14T14:57:34,299 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:57:34,299 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:57:34,299 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:57:34,299 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:57:34,299 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:57:34,300 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:57:34,300 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731596254299Disabling compacts and flushes for region at 1731596254299Disabling writes for close at 1731596254299Writing region close event to WAL at 1731596254299Closed at 1731596254299 2024-11-14T14:57:34,301 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:57:34,301 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T14:57:34,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T14:57:34,301 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.HRegionServer(746): ClusterId : 5edeb74e-79f3-471c-a397-1663b0e2c073 2024-11-14T14:57:34,301 DEBUG [RS:0;da1c6afcd1f9:36413 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T14:57:34,302 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:57:34,303 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T14:57:34,304 DEBUG [RS:0;da1c6afcd1f9:36413 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T14:57:34,304 DEBUG [RS:0;da1c6afcd1f9:36413 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T14:57:34,306 DEBUG [RS:0;da1c6afcd1f9:36413 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T14:57:34,306 DEBUG [RS:0;da1c6afcd1f9:36413 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@743fb570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=da1c6afcd1f9/172.17.0.2:0 2024-11-14T14:57:34,322 DEBUG [RS:0;da1c6afcd1f9:36413 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;da1c6afcd1f9:36413 2024-11-14T14:57:34,322 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T14:57:34,322 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T14:57:34,322 DEBUG [RS:0;da1c6afcd1f9:36413 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T14:57:34,322 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.HRegionServer(2659): reportForDuty to master=da1c6afcd1f9,46081,1731596254010 with port=36413, startcode=1731596254082 2024-11-14T14:57:34,323 DEBUG [RS:0;da1c6afcd1f9:36413 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T14:57:34,327 INFO [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55595, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T14:57:34,327 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46081 {}] master.ServerManager(363): Checking decommissioned status of RegionServer da1c6afcd1f9,36413,1731596254082 2024-11-14T14:57:34,327 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46081 {}] master.ServerManager(517): Registering regionserver=da1c6afcd1f9,36413,1731596254082 2024-11-14T14:57:34,329 DEBUG [RS:0;da1c6afcd1f9:36413 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4 2024-11-14T14:57:34,329 DEBUG [RS:0;da1c6afcd1f9:36413 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35151 2024-11-14T14:57:34,329 DEBUG [RS:0;da1c6afcd1f9:36413 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T14:57:34,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:57:34,331 DEBUG [RS:0;da1c6afcd1f9:36413 {}] zookeeper.ZKUtil(111): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/da1c6afcd1f9,36413,1731596254082 2024-11-14T14:57:34,331 WARN [RS:0;da1c6afcd1f9:36413 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T14:57:34,331 INFO [RS:0;da1c6afcd1f9:36413 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:57:34,332 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [da1c6afcd1f9,36413,1731596254082] 2024-11-14T14:57:34,332 DEBUG [RS:0;da1c6afcd1f9:36413 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/WALs/da1c6afcd1f9,36413,1731596254082 2024-11-14T14:57:34,340 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T14:57:34,342 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T14:57:34,342 INFO [RS:0;da1c6afcd1f9:36413 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T14:57:34,342 INFO [RS:0;da1c6afcd1f9:36413 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,342 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T14:57:34,343 INFO [RS:0;da1c6afcd1f9:36413 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T14:57:34,343 INFO [RS:0;da1c6afcd1f9:36413 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,343 DEBUG [RS:0;da1c6afcd1f9:36413 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:57:34,343 DEBUG [RS:0;da1c6afcd1f9:36413 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:57:34,344 DEBUG [RS:0;da1c6afcd1f9:36413 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:57:34,344 DEBUG [RS:0;da1c6afcd1f9:36413 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:57:34,344 DEBUG [RS:0;da1c6afcd1f9:36413 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:57:34,344 DEBUG [RS:0;da1c6afcd1f9:36413 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=2, maxPoolSize=2 2024-11-14T14:57:34,344 DEBUG [RS:0;da1c6afcd1f9:36413 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:57:34,344 DEBUG [RS:0;da1c6afcd1f9:36413 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:57:34,344 DEBUG [RS:0;da1c6afcd1f9:36413 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:57:34,344 DEBUG [RS:0;da1c6afcd1f9:36413 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:57:34,344 DEBUG [RS:0;da1c6afcd1f9:36413 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:57:34,344 DEBUG [RS:0;da1c6afcd1f9:36413 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/da1c6afcd1f9:0, corePoolSize=1, maxPoolSize=1 2024-11-14T14:57:34,344 DEBUG [RS:0;da1c6afcd1f9:36413 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:57:34,344 DEBUG [RS:0;da1c6afcd1f9:36413 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/da1c6afcd1f9:0, corePoolSize=3, maxPoolSize=3 2024-11-14T14:57:34,348 INFO [RS:0;da1c6afcd1f9:36413 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,348 INFO [RS:0;da1c6afcd1f9:36413 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,348 INFO [RS:0;da1c6afcd1f9:36413 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,348 INFO [RS:0;da1c6afcd1f9:36413 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,348 INFO [RS:0;da1c6afcd1f9:36413 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,348 INFO [RS:0;da1c6afcd1f9:36413 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,36413,1731596254082-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:57:34,364 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T14:57:34,365 INFO [RS:0;da1c6afcd1f9:36413 {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,36413,1731596254082-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,365 INFO [RS:0;da1c6afcd1f9:36413 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,365 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.Replication(171): da1c6afcd1f9,36413,1731596254082 started 2024-11-14T14:57:34,378 INFO [RS:0;da1c6afcd1f9:36413 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,378 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.HRegionServer(1482): Serving as da1c6afcd1f9,36413,1731596254082, RpcServer on da1c6afcd1f9/172.17.0.2:36413, sessionid=0x101a749708c0001 2024-11-14T14:57:34,379 DEBUG [RS:0;da1c6afcd1f9:36413 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T14:57:34,379 DEBUG [RS:0;da1c6afcd1f9:36413 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager da1c6afcd1f9,36413,1731596254082 2024-11-14T14:57:34,379 DEBUG [RS:0;da1c6afcd1f9:36413 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,36413,1731596254082' 2024-11-14T14:57:34,379 DEBUG [RS:0;da1c6afcd1f9:36413 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T14:57:34,379 DEBUG [RS:0;da1c6afcd1f9:36413 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T14:57:34,380 DEBUG [RS:0;da1c6afcd1f9:36413 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T14:57:34,380 DEBUG [RS:0;da1c6afcd1f9:36413 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T14:57:34,380 DEBUG [RS:0;da1c6afcd1f9:36413 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager da1c6afcd1f9,36413,1731596254082 2024-11-14T14:57:34,380 DEBUG [RS:0;da1c6afcd1f9:36413 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'da1c6afcd1f9,36413,1731596254082' 2024-11-14T14:57:34,380 DEBUG [RS:0;da1c6afcd1f9:36413 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T14:57:34,380 DEBUG [RS:0;da1c6afcd1f9:36413 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T14:57:34,381 DEBUG [RS:0;da1c6afcd1f9:36413 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T14:57:34,381 INFO [RS:0;da1c6afcd1f9:36413 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T14:57:34,381 INFO [RS:0;da1c6afcd1f9:36413 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T14:57:34,453 WARN [da1c6afcd1f9:46081 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-14T14:57:34,483 INFO [RS:0;da1c6afcd1f9:36413 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C36413%2C1731596254082, suffix=, logDir=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/WALs/da1c6afcd1f9,36413,1731596254082, archiveDir=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/oldWALs, maxLogs=32 2024-11-14T14:57:34,483 INFO [RS:0;da1c6afcd1f9:36413 {}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C36413%2C1731596254082.1731596254483 2024-11-14T14:57:34,489 INFO [RS:0;da1c6afcd1f9:36413 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/WALs/da1c6afcd1f9,36413,1731596254082/da1c6afcd1f9%2C36413%2C1731596254082.1731596254483 2024-11-14T14:57:34,490 DEBUG [RS:0;da1c6afcd1f9:36413 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39353:39353),(127.0.0.1/127.0.0.1:39351:39351)] 2024-11-14T14:57:34,604 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:34,604 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:34,704 DEBUG [da1c6afcd1f9:46081 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-14T14:57:34,704 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=da1c6afcd1f9,36413,1731596254082 2024-11-14T14:57:34,706 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,36413,1731596254082, state=OPENING 2024-11-14T14:57:34,708 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T14:57:34,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:57:34,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:57:34,710 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:57:34,710 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:57:34,710 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T14:57:34,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,36413,1731596254082}] 2024-11-14T14:57:34,863 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T14:57:34,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41475, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T14:57:34,869 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T14:57:34,869 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:57:34,870 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=da1c6afcd1f9%2C36413%2C1731596254082.meta, suffix=.meta, logDir=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/WALs/da1c6afcd1f9,36413,1731596254082, archiveDir=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/oldWALs, maxLogs=32 2024-11-14T14:57:34,871 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor da1c6afcd1f9%2C36413%2C1731596254082.meta.1731596254871.meta 2024-11-14T14:57:34,876 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/WALs/da1c6afcd1f9,36413,1731596254082/da1c6afcd1f9%2C36413%2C1731596254082.meta.1731596254871.meta 2024-11-14T14:57:34,881 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39351:39351),(127.0.0.1/127.0.0.1:39353:39353)] 2024-11-14T14:57:34,882 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T14:57:34,883 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T14:57:34,883 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T14:57:34,883 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T14:57:34,883 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T14:57:34,883 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T14:57:34,883 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T14:57:34,883 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T14:57:34,885 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T14:57:34,886 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T14:57:34,886 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:57:34,887 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:57:34,887 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T14:57:34,887 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T14:57:34,888 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:57:34,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:57:34,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T14:57:34,889 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T14:57:34,889 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:57:34,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:57:34,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T14:57:34,890 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T14:57:34,890 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T14:57:34,891 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T14:57:34,891 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T14:57:34,892 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/data/hbase/meta/1588230740 2024-11-14T14:57:34,893 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/data/hbase/meta/1588230740 2024-11-14T14:57:34,895 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T14:57:34,895 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T14:57:34,895 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T14:57:34,897 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T14:57:34,898 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=746821, jitterRate=-0.05036872625350952}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T14:57:34,898 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T14:57:34,898 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731596254883Writing region info on filesystem at 1731596254883Initializing all the Stores at 1731596254884 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596254884Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596254885 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731596254885Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731596254885Cleaning up temporary data from old regions at 1731596254895 (+10 ms)Running coprocessor post-open hooks at 1731596254898 (+3 ms)Region opened successfully at 1731596254898 2024-11-14T14:57:34,899 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731596254863 2024-11-14T14:57:34,902 DEBUG [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T14:57:34,902 INFO [RS_OPEN_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T14:57:34,903 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=da1c6afcd1f9,36413,1731596254082 2024-11-14T14:57:34,904 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as da1c6afcd1f9,36413,1731596254082, state=OPEN 2024-11-14T14:57:34,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:57:34,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T14:57:34,908 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,36413,1731596254082 2024-11-14T14:57:34,908 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:57:34,908 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T14:57:34,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T14:57:34,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=da1c6afcd1f9,36413,1731596254082 in 198 msec 2024-11-14T14:57:34,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T14:57:34,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 610 msec 2024-11-14T14:57:34,915 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T14:57:34,915 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T14:57:34,916 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:57:34,916 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,36413,1731596254082, seqNum=-1] 2024-11-14T14:57:34,916 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:57:34,918 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57153, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:57:34,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 664 msec 2024-11-14T14:57:34,923 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731596254923, completionTime=-1 2024-11-14T14:57:34,923 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-14T14:57:34,923 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-14T14:57:34,926 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-14T14:57:34,926 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731596314926 2024-11-14T14:57:34,926 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731596374926 2024-11-14T14:57:34,926 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-14T14:57:34,926 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,46081,1731596254010-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,926 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,46081,1731596254010-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,926 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,46081,1731596254010-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,926 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-da1c6afcd1f9:46081, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,926 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,927 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:34,928 DEBUG [master/da1c6afcd1f9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T14:57:34,930 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.813sec 2024-11-14T14:57:34,930 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T14:57:34,930 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T14:57:34,930 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T14:57:34,930 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T14:57:34,930 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T14:57:34,930 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,46081,1731596254010-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T14:57:34,930 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,46081,1731596254010-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T14:57:34,933 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T14:57:34,933 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T14:57:34,933 INFO [master/da1c6afcd1f9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=da1c6afcd1f9,46081,1731596254010-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T14:57:35,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b3ba97d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:57:35,001 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request da1c6afcd1f9,46081,-1 for getting cluster id 2024-11-14T14:57:35,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T14:57:35,004 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5edeb74e-79f3-471c-a397-1663b0e2c073' 2024-11-14T14:57:35,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T14:57:35,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5edeb74e-79f3-471c-a397-1663b0e2c073" 2024-11-14T14:57:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73fc411, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:57:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [da1c6afcd1f9,46081,-1] 2024-11-14T14:57:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T14:57:35,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:57:35,006 INFO [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T14:57:35,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b4a64d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T14:57:35,007 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T14:57:35,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=da1c6afcd1f9,36413,1731596254082, seqNum=-1] 2024-11-14T14:57:35,008 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T14:57:35,009 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58830, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T14:57:35,011 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=da1c6afcd1f9,46081,1731596254010 2024-11-14T14:57:35,011 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T14:57:35,013 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-14T14:57:35,014 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-14T14:57:35,015 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/WALs/test.com,8080,1, archiveDir=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/oldWALs, maxLogs=32 2024-11-14T14:57:35,016 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731596255016 2024-11-14T14:57:35,020 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/WALs/test.com,8080,1/test.com%2C8080%2C1.1731596255016 2024-11-14T14:57:35,022 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39353:39353),(127.0.0.1/127.0.0.1:39351:39351)] 2024-11-14T14:57:35,026 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731596255026 2024-11-14T14:57:35,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,034 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/WALs/test.com,8080,1/test.com%2C8080%2C1.1731596255016 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/WALs/test.com,8080,1/test.com%2C8080%2C1.1731596255026 2024-11-14T14:57:35,035 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39353:39353),(127.0.0.1/127.0.0.1:39351:39351)] 2024-11-14T14:57:35,035 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/WALs/test.com,8080,1/test.com%2C8080%2C1.1731596255016 is not closed yet, will try archiving it next time 2024-11-14T14:57:35,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44669 is added to blk_1073741835_1011 (size=93) 2024-11-14T14:57:35,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37481 is added to blk_1073741835_1011 (size=93) 2024-11-14T14:57:35,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,037 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/WALs/test.com,8080,1/test.com%2C8080%2C1.1731596255016 to hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/oldWALs/test.com%2C8080%2C1.1731596255016 2024-11-14T14:57:35,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44669 is added to blk_1073741836_1012 (size=93) 2024-11-14T14:57:35,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37481 is added to blk_1073741836_1012 (size=93) 2024-11-14T14:57:35,047 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/oldWALs 2024-11-14T14:57:35,047 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1731596255026) 2024-11-14T14:57:35,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T14:57:35,047 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:57:35,047 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:57:35,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:57:35,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:57:35,048 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T14:57:35,048 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T14:57:35,048 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=476314802, stopped=false 2024-11-14T14:57:35,048 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=da1c6afcd1f9,46081,1731596254010 2024-11-14T14:57:35,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:57:35,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T14:57:35,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:57:35,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:57:35,049 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:57:35,050 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T14:57:35,050 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:57:35,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:57:35,050 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'da1c6afcd1f9,36413,1731596254082' ***** 2024-11-14T14:57:35,050 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T14:57:35,050 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T14:57:35,050 INFO [RS:0;da1c6afcd1f9:36413 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T14:57:35,050 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T14:57:35,050 INFO [RS:0;da1c6afcd1f9:36413 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T14:57:35,051 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.HRegionServer(959): stopping server da1c6afcd1f9,36413,1731596254082 2024-11-14T14:57:35,051 INFO [RS:0;da1c6afcd1f9:36413 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:57:35,051 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:57:35,051 INFO [RS:0;da1c6afcd1f9:36413 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;da1c6afcd1f9:36413. 2024-11-14T14:57:35,051 DEBUG [RS:0;da1c6afcd1f9:36413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T14:57:35,051 DEBUG [RS:0;da1c6afcd1f9:36413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:57:35,051 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T14:57:35,051 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T14:57:35,051 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T14:57:35,051 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T14:57:35,051 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T14:57:35,060 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-14T14:57:35,060 DEBUG [RS:0;da1c6afcd1f9:36413 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-14T14:57:35,060 DEBUG [RS:0;da1c6afcd1f9:36413 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-14T14:57:35,060 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T14:57:35,060 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T14:57:35,060 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T14:57:35,060 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T14:57:35,060 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T14:57:35,060 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-11-14T14:57:35,085 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/data/hbase/meta/1588230740/.tmp/ns/01444edc51314db1943a2eae54b97ef0 is 43, key is default/ns:d/1731596254918/Put/seqid=0 2024-11-14T14:57:35,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44669 is added to blk_1073741837_1013 (size=5153) 2024-11-14T14:57:35,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37481 is added to blk_1073741837_1013 (size=5153) 2024-11-14T14:57:35,095 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/data/hbase/meta/1588230740/.tmp/ns/01444edc51314db1943a2eae54b97ef0 2024-11-14T14:57:35,102 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/data/hbase/meta/1588230740/.tmp/ns/01444edc51314db1943a2eae54b97ef0 as hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/data/hbase/meta/1588230740/ns/01444edc51314db1943a2eae54b97ef0 2024-11-14T14:57:35,107 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/data/hbase/meta/1588230740/ns/01444edc51314db1943a2eae54b97ef0, entries=2, sequenceid=6, filesize=5.0 K 2024-11-14T14:57:35,108 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 48ms, sequenceid=6, compaction requested=false 2024-11-14T14:57:35,108 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T14:57:35,113 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-14T14:57:35,114 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T14:57:35,114 INFO [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T14:57:35,114 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731596255060Running coprocessor pre-close hooks at 1731596255060Disabling compacts and flushes for region at 1731596255060Disabling writes for close at 1731596255060Obtaining lock to block concurrent updates at 1731596255060Preparing flush snapshotting stores in 1588230740 at 1731596255060Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731596255061 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731596255062 (+1 ms)Flushing 1588230740/ns: creating writer at 1731596255062Flushing 1588230740/ns: appending metadata at 1731596255084 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1731596255084Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bb58575: reopening flushed file at 1731596255101 (+17 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 48ms, sequenceid=6, compaction requested=false at 1731596255108 (+7 ms)Writing region close event to WAL at 1731596255110 (+2 ms)Running coprocessor post-close hooks at 1731596255114 (+4 ms)Closed at 1731596255114 2024-11-14T14:57:35,114 DEBUG [RS_CLOSE_META-regionserver/da1c6afcd1f9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T14:57:35,260 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.HRegionServer(976): stopping server da1c6afcd1f9,36413,1731596254082; all regions closed. 2024-11-14T14:57:35,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37481 is added to blk_1073741834_1010 (size=1152) 2024-11-14T14:57:35,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44669 is added to blk_1073741834_1010 (size=1152) 2024-11-14T14:57:35,267 DEBUG [RS:0;da1c6afcd1f9:36413 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/oldWALs 2024-11-14T14:57:35,267 INFO [RS:0;da1c6afcd1f9:36413 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog da1c6afcd1f9%2C36413%2C1731596254082.meta:.meta(num 1731596254871) 2024-11-14T14:57:35,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44669 is added to blk_1073741833_1009 (size=93) 2024-11-14T14:57:35,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37481 is added to blk_1073741833_1009 (size=93) 2024-11-14T14:57:35,278 DEBUG [RS:0;da1c6afcd1f9:36413 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/oldWALs 2024-11-14T14:57:35,278 INFO [RS:0;da1c6afcd1f9:36413 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog da1c6afcd1f9%2C36413%2C1731596254082:(num 1731596254483) 2024-11-14T14:57:35,278 DEBUG [RS:0;da1c6afcd1f9:36413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T14:57:35,278 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T14:57:35,278 INFO [RS:0;da1c6afcd1f9:36413 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:57:35,278 INFO [RS:0;da1c6afcd1f9:36413 {}] hbase.ChoreService(370): Chore service for: regionserver/da1c6afcd1f9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T14:57:35,278 INFO [RS:0;da1c6afcd1f9:36413 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:57:35,278 INFO [regionserver/da1c6afcd1f9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:57:35,278 INFO [RS:0;da1c6afcd1f9:36413 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36413 2024-11-14T14:57:35,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T14:57:35,281 INFO [RS:0;da1c6afcd1f9:36413 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:57:35,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/da1c6afcd1f9,36413,1731596254082 2024-11-14T14:57:35,282 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [da1c6afcd1f9,36413,1731596254082] 2024-11-14T14:57:35,284 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/da1c6afcd1f9,36413,1731596254082 already deleted, retry=false 2024-11-14T14:57:35,284 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; da1c6afcd1f9,36413,1731596254082 expired; onlineServers=0 2024-11-14T14:57:35,284 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'da1c6afcd1f9,46081,1731596254010' ***** 2024-11-14T14:57:35,284 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T14:57:35,284 INFO [M:0;da1c6afcd1f9:46081 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T14:57:35,284 INFO [M:0;da1c6afcd1f9:46081 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T14:57:35,284 DEBUG [M:0;da1c6afcd1f9:46081 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T14:57:35,284 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T14:57:35,284 DEBUG [M:0;da1c6afcd1f9:46081 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T14:57:35,284 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731596254264 {}] cleaner.HFileCleaner(306): Exit Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.small.0-1731596254264,5,FailOnTimeoutGroup] 2024-11-14T14:57:35,284 DEBUG [master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731596254264 {}] cleaner.HFileCleaner(306): Exit Thread[master/da1c6afcd1f9:0:becomeActiveMaster-HFileCleaner.large.0-1731596254264,5,FailOnTimeoutGroup] 2024-11-14T14:57:35,284 INFO [M:0;da1c6afcd1f9:46081 {}] hbase.ChoreService(370): Chore service for: master/da1c6afcd1f9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T14:57:35,285 INFO [M:0;da1c6afcd1f9:46081 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T14:57:35,285 DEBUG [M:0;da1c6afcd1f9:46081 {}] master.HMaster(1795): Stopping service threads 2024-11-14T14:57:35,285 INFO [M:0;da1c6afcd1f9:46081 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T14:57:35,285 INFO [M:0;da1c6afcd1f9:46081 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T14:57:35,285 INFO [M:0;da1c6afcd1f9:46081 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T14:57:35,285 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T14:57:35,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T14:57:35,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T14:57:35,286 DEBUG [M:0;da1c6afcd1f9:46081 {}] zookeeper.ZKUtil(347): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T14:57:35,286 WARN [M:0;da1c6afcd1f9:46081 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T14:57:35,287 INFO [M:0;da1c6afcd1f9:46081 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/.lastflushedseqids 2024-11-14T14:57:35,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44669 is added to blk_1073741838_1014 (size=99) 2024-11-14T14:57:35,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37481 is added to blk_1073741838_1014 (size=99) 2024-11-14T14:57:35,295 INFO [M:0;da1c6afcd1f9:46081 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T14:57:35,295 INFO [M:0;da1c6afcd1f9:46081 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T14:57:35,296 DEBUG [M:0;da1c6afcd1f9:46081 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T14:57:35,296 INFO [M:0;da1c6afcd1f9:46081 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:57:35,296 DEBUG [M:0;da1c6afcd1f9:46081 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:57:35,296 DEBUG [M:0;da1c6afcd1f9:46081 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T14:57:35,296 DEBUG [M:0;da1c6afcd1f9:46081 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:57:35,296 INFO [M:0;da1c6afcd1f9:46081 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-11-14T14:57:35,312 DEBUG [M:0;da1c6afcd1f9:46081 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a9ec7d3e45bd4c8095c35c72383732d6 is 82, key is hbase:meta,,1/info:regioninfo/1731596254903/Put/seqid=0 2024-11-14T14:57:35,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44669 is added to blk_1073741839_1015 (size=5672) 2024-11-14T14:57:35,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37481 is added to blk_1073741839_1015 (size=5672) 2024-11-14T14:57:35,317 INFO [M:0;da1c6afcd1f9:46081 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a9ec7d3e45bd4c8095c35c72383732d6 2024-11-14T14:57:35,343 DEBUG [M:0;da1c6afcd1f9:46081 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a78bf72c95104be4a78a46fb77bc5b5c is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731596254922/Put/seqid=0 2024-11-14T14:57:35,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37481 is added to blk_1073741840_1016 (size=5275) 2024-11-14T14:57:35,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44669 is added to blk_1073741840_1016 (size=5275) 2024-11-14T14:57:35,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:57:35,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36413-0x101a749708c0001, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:57:35,383 INFO [RS:0;da1c6afcd1f9:36413 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:57:35,383 INFO [RS:0;da1c6afcd1f9:36413 {}] regionserver.HRegionServer(1031): Exiting; stopping=da1c6afcd1f9,36413,1731596254082; zookeeper connection closed. 2024-11-14T14:57:35,383 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@123c7620 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@123c7620 2024-11-14T14:57:35,383 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-14T14:57:35,605 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,34075,1731596060544/da1c6afcd1f9%2C34075%2C1731596060544.1731596060811 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:35,605 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:35595/user/jenkins/test-data/e566ce97-8406-4d70-57a9-2fe0f4982eb7/WALs/da1c6afcd1f9,40527,1731596059482/da1c6afcd1f9%2C40527%2C1731596059482.meta.1731596060396.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-14T14:57:35,756 INFO [M:0;da1c6afcd1f9:46081 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a78bf72c95104be4a78a46fb77bc5b5c 2024-11-14T14:57:35,777 DEBUG [M:0;da1c6afcd1f9:46081 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/92040bab1ba34672bbeba4fc8253c93c is 69, key is da1c6afcd1f9,36413,1731596254082/rs:state/1731596254327/Put/seqid=0 2024-11-14T14:57:35,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37481 is added to blk_1073741841_1017 (size=5156) 2024-11-14T14:57:35,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44669 is added to blk_1073741841_1017 (size=5156) 2024-11-14T14:57:35,782 INFO [M:0;da1c6afcd1f9:46081 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/92040bab1ba34672bbeba4fc8253c93c 2024-11-14T14:57:35,804 DEBUG [M:0;da1c6afcd1f9:46081 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e970227717ad4973989e93f714416792 is 52, key is load_balancer_on/state:d/1731596255013/Put/seqid=0 2024-11-14T14:57:35,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44669 is added to blk_1073741842_1018 (size=5056) 2024-11-14T14:57:35,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37481 is added to blk_1073741842_1018 (size=5056) 2024-11-14T14:57:35,810 INFO [M:0;da1c6afcd1f9:46081 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e970227717ad4973989e93f714416792 2024-11-14T14:57:35,814 DEBUG [M:0;da1c6afcd1f9:46081 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a9ec7d3e45bd4c8095c35c72383732d6 as hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a9ec7d3e45bd4c8095c35c72383732d6 2024-11-14T14:57:35,818 INFO [M:0;da1c6afcd1f9:46081 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a9ec7d3e45bd4c8095c35c72383732d6, entries=8, sequenceid=29, filesize=5.5 K 2024-11-14T14:57:35,819 DEBUG [M:0;da1c6afcd1f9:46081 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a78bf72c95104be4a78a46fb77bc5b5c as hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a78bf72c95104be4a78a46fb77bc5b5c 2024-11-14T14:57:35,822 INFO [M:0;da1c6afcd1f9:46081 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a78bf72c95104be4a78a46fb77bc5b5c, entries=3, sequenceid=29, filesize=5.2 K 2024-11-14T14:57:35,823 DEBUG [M:0;da1c6afcd1f9:46081 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/92040bab1ba34672bbeba4fc8253c93c as hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/92040bab1ba34672bbeba4fc8253c93c 2024-11-14T14:57:35,828 INFO [M:0;da1c6afcd1f9:46081 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/92040bab1ba34672bbeba4fc8253c93c, entries=1, sequenceid=29, filesize=5.0 K 2024-11-14T14:57:35,829 DEBUG [M:0;da1c6afcd1f9:46081 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e970227717ad4973989e93f714416792 as hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e970227717ad4973989e93f714416792 2024-11-14T14:57:35,833 INFO [M:0;da1c6afcd1f9:46081 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35151/user/jenkins/test-data/4989fc07-071d-2f36-a3fb-8dc7d9261ac4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e970227717ad4973989e93f714416792, entries=1, sequenceid=29, filesize=4.9 K 2024-11-14T14:57:35,834 INFO [M:0;da1c6afcd1f9:46081 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 538ms, sequenceid=29, compaction requested=false 2024-11-14T14:57:35,841 INFO [M:0;da1c6afcd1f9:46081 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T14:57:35,841 DEBUG [M:0;da1c6afcd1f9:46081 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731596255296Disabling compacts and flushes for region at 1731596255296Disabling writes for close at 1731596255296Obtaining lock to block concurrent updates at 1731596255296Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731596255296Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731596255296Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731596255297 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731596255297Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731596255312 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731596255312Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731596255322 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731596255343 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731596255343Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731596255760 (+417 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731596255776 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731596255776Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731596255786 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731596255803 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731596255803Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ea4d37: reopening flushed file at 1731596255813 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8ba146a: reopening flushed file at 1731596255818 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7041c68: reopening flushed file at 1731596255823 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21b50ee6: reopening flushed file at 1731596255828 (+5 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 538ms, sequenceid=29, compaction requested=false at 1731596255834 (+6 ms)Writing region close event to WAL at 1731596255841 (+7 ms)Closed at 1731596255841 2024-11-14T14:57:35,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T14:57:35,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44669 is added to blk_1073741830_1006 (size=10311) 2024-11-14T14:57:35,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37481 is added to blk_1073741830_1006 (size=10311) 2024-11-14T14:57:35,845 INFO [M:0;da1c6afcd1f9:46081 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T14:57:35,845 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T14:57:35,845 INFO [M:0;da1c6afcd1f9:46081 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46081 2024-11-14T14:57:35,845 INFO [M:0;da1c6afcd1f9:46081 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T14:57:35,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:57:35,947 INFO [M:0;da1c6afcd1f9:46081 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T14:57:35,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46081-0x101a749708c0000, quorum=127.0.0.1:50420, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T14:57:35,949 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d78da71{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:57:35,950 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@39d5f486{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:57:35,950 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:57:35,950 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@536a5c19{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:57:35,950 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@231297c7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/hadoop.log.dir/,STOPPED} 2024-11-14T14:57:35,952 WARN [BP-1130588054-172.17.0.2-1731596253125 heartbeating to localhost/127.0.0.1:35151 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:57:35,952 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:57:35,952 WARN [BP-1130588054-172.17.0.2-1731596253125 heartbeating to localhost/127.0.0.1:35151 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1130588054-172.17.0.2-1731596253125 (Datanode Uuid 732513ac-b026-4d46-8f04-ff2b213483cf) service to localhost/127.0.0.1:35151 2024-11-14T14:57:35,952 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:57:35,952 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/cluster_0e145edd-8855-07ba-125d-fb4f19b1c6fa/data/data3/current/BP-1130588054-172.17.0.2-1731596253125 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:57:35,953 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/cluster_0e145edd-8855-07ba-125d-fb4f19b1c6fa/data/data4/current/BP-1130588054-172.17.0.2-1731596253125 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:57:35,953 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:57:35,955 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3698cd86{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T14:57:35,955 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@418bce8c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:57:35,955 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:57:35,955 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@35a6030c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:57:35,955 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@677f535e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/hadoop.log.dir/,STOPPED} 2024-11-14T14:57:35,957 WARN [BP-1130588054-172.17.0.2-1731596253125 heartbeating to localhost/127.0.0.1:35151 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T14:57:35,957 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T14:57:35,957 WARN [BP-1130588054-172.17.0.2-1731596253125 heartbeating to localhost/127.0.0.1:35151 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1130588054-172.17.0.2-1731596253125 (Datanode Uuid d8011282-0532-4b70-b48e-001dd156073c) service to localhost/127.0.0.1:35151 2024-11-14T14:57:35,957 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T14:57:35,957 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/cluster_0e145edd-8855-07ba-125d-fb4f19b1c6fa/data/data1/current/BP-1130588054-172.17.0.2-1731596253125 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:57:35,957 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/cluster_0e145edd-8855-07ba-125d-fb4f19b1c6fa/data/data2/current/BP-1130588054-172.17.0.2-1731596253125 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T14:57:35,958 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T14:57:35,963 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1b488929{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T14:57:35,964 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@54644e01{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T14:57:35,964 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T14:57:35,964 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@442ec576{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T14:57:35,964 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@30964c14{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2628b3be-acfe-b019-b4b9-412bbb96d184/hadoop.log.dir/,STOPPED} 2024-11-14T14:57:35,970 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T14:57:35,983 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-14T14:57:35,993 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=270 (was 232) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35151 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35151 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:35151 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35151 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35151 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:35151 from jenkins.hfs.7 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:35151 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: globalEventExecutor-1-21 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//io.netty.util.concurrent.GlobalEventExecutor.takeTask(GlobalEventExecutor.java:113) app//io.netty.util.concurrent.GlobalEventExecutor$TaskRunner.run(GlobalEventExecutor.java:259) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:35151 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=541 (was 518) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=175 (was 173) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8266 (was 8501)